From f868f5f723c247cc8e8201ea08f9d906bc0fa709 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 16 May 2018 08:17:30 +0000
Subject: [PATCH 01/95] WIP prototype sysTest

Requires:
robotFramework - the system test framework
pyautogui - for controlling keyboard/mouse

To install:
pip install robotframework
pip install pyautogui
---
 tests/system/initial.robot | 17 +++++++++++++++++
 tests/system/sendKey.py    | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 tests/system/initial.robot
 create mode 100644 tests/system/sendKey.py

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
new file mode 100644
index 00000000000..66769b92051
--- /dev/null
+++ b/tests/system/initial.robot
@@ -0,0 +1,17 @@
+*** Settings ***
+Documentation    An example test suite documentation with *some* _formatting_.
+...              See test documentation for more documentation examples.
+Library       OperatingSystem
+Library       Process
+Library       sendKey.py
+
+*** Test Cases ***
+
+Can Start NVDA
+    ${nvdaHandle} =  Start Process  python nvda.pyw --debug-logging  cwd=source  shell=true
+    Process Should Be Running  ${nvdaHandle}
+    sleep  5
+    send quit NVDA keys
+    ${nvdaResult} =  Wait For Process  ${nvdaHandle}
+    Should Be Equal  ${nvdaResult.stdout}  Hello, world!
+
diff --git a/tests/system/sendKey.py b/tests/system/sendKey.py
new file mode 100644
index 00000000000..60c2c2e5bbb
--- /dev/null
+++ b/tests/system/sendKey.py
@@ -0,0 +1,32 @@
+import pyautogui
+
+"""
+['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(',
+')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7',
+'8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`',
+'a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~',
+'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace',
+'browserback', 'browserfavorites', 'browserforward', 'browserhome',
+'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear',
+'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete',
+'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10',
+'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20',
+'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9',
+'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja',
+'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail',
+'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack',
+'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6',
+'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn',
+'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn',
+'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator',
+'shift', 'shiftleft', 'shiftright', 'sleep', 'space', 'stop', 'subtract', 'tab',
+'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen',
+'command', 'option', 'optionleft', 'optionright']
+"""
+
+def send_quit_NVDA_keys():
+    """
+    Sends Caps+q
+    """
+    pyautogui.hotkey('capslock', 'q')
\ No newline at end of file

From e9a2af7ce55db8d1745881bb6a047d6f37b10163 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 16 May 2018 08:34:16 +0000
Subject: [PATCH 02/95] WIP include flaky way of exiting

Ensure this is run from a cmd prompt. Other shells may cause issues,
for instance, NVDA does not get correct focus when run from cygwin.
---
 tests/system/initial.robot | 4 +++-
 tests/system/sendKey.py    | 8 +++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 66769b92051..e4517f2977e 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -8,10 +8,12 @@ Library       sendKey.py
 *** Test Cases ***
 
 Can Start NVDA
-    ${nvdaHandle} =  Start Process  python nvda.pyw --debug-logging  cwd=source  shell=true
+    ${nvdaHandle} =  Start Process  pythonw nvda.pyw --debug-logging  cwd=source  shell=true
     Process Should Be Running  ${nvdaHandle}
     sleep  5
     send quit NVDA keys
+    sleep  1
+    send enter key
     ${nvdaResult} =  Wait For Process  ${nvdaHandle}
     Should Be Equal  ${nvdaResult.stdout}  Hello, world!
 
diff --git a/tests/system/sendKey.py b/tests/system/sendKey.py
index 60c2c2e5bbb..9cd51dca952 100644
--- a/tests/system/sendKey.py
+++ b/tests/system/sendKey.py
@@ -29,4 +29,10 @@ def send_quit_NVDA_keys():
     """
     Sends Caps+q
     """
-    pyautogui.hotkey('capslock', 'q')
\ No newline at end of file
+    pyautogui.hotkey('capslock', 'q')
+
+def send_enter_key():
+    """
+    Sends ENTER
+    """
+    pyautogui.hotkey('enter')
\ No newline at end of file

From 1fb27cbb6936e1dfb2083eee2a6a980e01cbbdf8 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Thu, 17 May 2018 11:00:53 +0000
Subject: [PATCH 03/95] WIP System testing

Remote server implemenation and external abstracted library.
---
 source/core.py                |  6 +++
 tests/system/initial.robot    | 17 ++++----
 tests/system/nvdaRobotLib.py  | 74 +++++++++++++++++++++++++++++++++++
 tests/system/systemTestSpy.py | 49 +++++++++++++++++++++++
 4 files changed, 136 insertions(+), 10 deletions(-)
 create mode 100644 tests/system/nvdaRobotLib.py
 create mode 100644 tests/system/systemTestSpy.py

diff --git a/source/core.py b/source/core.py
index 4fae280b0a5..5ed206388f7 100644
--- a/source/core.py
+++ b/source/core.py
@@ -29,6 +29,11 @@
 from logHandler import log
 import addonHandler
 
+import extensionPoints
+
+# inform those who want to know that NVDA has finished starting up.
+postNvdaStartup = extensionPoints.Action()
+
 PUMP_MAX_DELAY = 10
 
 #: The thread identifier of the main thread.
@@ -468,6 +473,7 @@ def Notify(self):
 		log.debug("initializing updateCheck")
 		updateCheck.initialize()
 	log.info("NVDA initialized")
+	postNvdaStartup.notify()
 
 	log.debug("entering wx application main loop")
 	app.MainLoop()
diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index e4517f2977e..863f6b792dc 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -1,19 +1,16 @@
 *** Settings ***
-Documentation    An example test suite documentation with *some* _formatting_.
-...              See test documentation for more documentation examples.
+Documentation    Basic *NVDA* and _RobotFramework_ tests
+...              Starts NVDA and exits.
+...              Run with python -m robot tests/system/initial.robot in CMD.
 Library       OperatingSystem
 Library       Process
 Library       sendKey.py
+Library       nvdaRobotLib.py
 
 *** Test Cases ***
 
-Can Start NVDA
-    ${nvdaHandle} =  Start Process  pythonw nvda.pyw --debug-logging  cwd=source  shell=true
-    Process Should Be Running  ${nvdaHandle}
-    sleep  5
-    send quit NVDA keys
-    sleep  1
-    send enter key
-    ${nvdaResult} =  Wait For Process  ${nvdaHandle}
+Can Start and exit NVDA
+    ${nvdaProcessHandle} =  start NVDA
+    ${nvdaResult} =  quit NVDA
     Should Be Equal  ${nvdaResult.stdout}  Hello, world!
 
diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
new file mode 100644
index 00000000000..e16470c92fe
--- /dev/null
+++ b/tests/system/nvdaRobotLib.py
@@ -0,0 +1,74 @@
+from robot.libraries.BuiltIn import BuiltIn
+from robot.libraries.OperatingSystem import OperatingSystem
+from robot.libraries.Process import Process
+import sendKey
+
+builtIn = BuiltIn()
+os = OperatingSystem()
+process = Process()
+
+class nvdaRobotLib(object):
+
+	def __init__(self):
+		self.nvdaSpy = None
+		self.nvdaHandle = None
+
+
+	def copy_in_system_test_spy(self):
+		"""Equiv robot text:
+		Copy File  tests/system/systemTestSpy.py  source/globalPlugins/
+		"""
+		os.copy_file("tests/system/systemTestSpy.py", "source/globalPlugins/")
+
+
+	def _startNVDAProcess(self):
+		"""Equiv robot text:
+		Start Process  pythonw nvda.pyw --debug-logging  cwd=source  shell=true  alias=nvdaAlias
+		"""
+		self.nvdaHandle = handle = process.start_process(
+			"pythonw nvda.pyw --debug-logging",
+			cwd='source',
+			shell=True,
+			alias='nvdaAlias'
+		)
+		return handle
+
+
+	def _connectToRemoteServer(self):
+		"""Equiv robot text:
+		Import Library  Remote         WITH NAME    nvdaSpy
+		"""
+		builtIn.import_library(
+				"Remote",
+				'http://127.0.0.1:8270',
+				"WITH NAME",
+				"nvdaSpy"
+			)
+		self.nvdaSpy = builtIn.get_library_instance("nvdaSpy")
+
+
+	def start_NVDA(self):
+		self.copy_in_system_test_spy()
+		nvdaProcessHandle = self._startNVDAProcess()
+		process.process_should_be_running(nvdaProcessHandle)
+		self._connectToRemoteServer()
+		self.wait_for_NVDA_startup_to_complete()
+		return nvdaProcessHandle
+
+
+	def wait_for_NVDA_startup_to_complete(self):
+		while not self.nvdaSpy.run_keyword("is_NVDA_startup_complete", [], {}):
+			builtIn.sleep(0.1)
+
+	def quit_NVDA(self):
+		"""send quit NVDA keys
+			sleep  1
+			send enter key
+			nvdaSpy.Stop Remote Server
+			Wait For Process  nvdaAlias
+		"""
+		sendKey.send_quit_NVDA_keys()
+		builtIn.sleep(1.0)
+		sendKey.send_enter_key()
+		self.nvdaSpy.run_keyword("stop_remote_server", [], {})
+		return process.wait_for_process(self.nvdaHandle)
diff --git a/tests/system/systemTestSpy.py b/tests/system/systemTestSpy.py
new file mode 100644
index 00000000000..0d5e9db0745
--- /dev/null
+++ b/tests/system/systemTestSpy.py
@@ -0,0 +1,49 @@
+import globalPluginHandler
+import signal
+import threading
+from robotremoteserver import RobotRemoteServer
+from logHandler import log
+
+class SystemTestSpy:
+	def __init__(self):
+		self._nvdaStartupComplete = False
+		from core import postNvdaStartup
+		postNvdaStartup.register(self._onNvdaStartupComplete)
+
+	def _onNvdaStartupComplete(self):
+		self._nvdaStartupComplete = True
+
+	def is_NVDA_startup_complete(self):
+		log.debug("Got startup complete action")
+		return self._nvdaStartupComplete
+
+
+class SystemTestSpyServer(object):
+	def __init__(self):
+		self._server = None
+
+	def start(self):
+		log.debug("TestSpyPlugin started")
+		server = self._server = RobotRemoteServer(
+			SystemTestSpy(),  # provides actual spy behaviour
+			port=8270,  # Could use 0 for auto port selection, which can be written out to a file / command line
+			serve=False  # we want to start this serving on another thread so as not to block.
+		)
+		log.debug("Server address: {}".format(server.server_address))
+		signal.signal(signal.SIGINT, lambda signum, frame: server.stop())
+		server_thread = threading.Thread(target=server.serve)
+		server_thread.start()
+
+	def stop(self):
+		self._server.stop()
+
+
+class GlobalPlugin(globalPluginHandler.GlobalPlugin):
+
+	def __init__(self):
+		super(GlobalPlugin, self).__init__()
+		self._testSpy = SystemTestSpyServer()
+		self._testSpy.start()
+
+	__gestures = {
+	}
\ No newline at end of file

From 5c83124678e95c98b755f3c65fb60bcbbfd1ab12 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 7 Jun 2018 16:29:46 +1000
Subject: [PATCH 04/95] First try at including system tests in appveyor.

---
 appveyor.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/appveyor.yml b/appveyor.yml
index ab0b60a85d6..b6b81a3c25b 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -97,6 +97,20 @@ build_script:
  - 7z a -tzip -r ..\output\symbols.zip *.dl_ *.ex_ *.pd_
  - cd ..
 
+before_test:
+ - py -m pip install robotframework
+ - py -m pip install robotremoteserver
+ - py -m pip install pyautogui
+ - mkdir testResults
+
+test_script:
+ - py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot
+ 
+after_test:
+ - ps: |
+     $wc = New-Object 'System.Net.WebClient'
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/xunit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests_xunit.xml))
+ 
 artifacts:
  - path: output\*
  - path: output\*\*

From dd9f3e77022728c154c89a3b5da35a1b276d825e Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 7 Jun 2018 17:07:37 +1000
Subject: [PATCH 05/95] Disable pyAutoGUI's failsafe check as appeyor moves the
 mouse???

---
 tests/system/sendKey.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/system/sendKey.py b/tests/system/sendKey.py
index 9cd51dca952..83102cfb1be 100644
--- a/tests/system/sendKey.py
+++ b/tests/system/sendKey.py
@@ -1,4 +1,5 @@
 import pyautogui
+pyautogui.FAILSAFE=False
 
 """
 ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(',

From 1e6e265cb44120f51045839f5aedf462ea110059 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 7 Jun 2018 22:52:47 +1000
Subject: [PATCH 06/95] initial.robot: break robot to temporarily bipass
 freeze.

---
 tests/system/initial.robot | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 863f6b792dc..e8d885a9f67 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -10,7 +10,6 @@ Library       nvdaRobotLib.py
 *** Test Cases ***
 
 Can Start and exit NVDA
-    ${nvdaProcessHandle} =  start NVDA
     ${nvdaResult} =  quit NVDA
     Should Be Equal  ${nvdaResult.stdout}  Hello, world!
 

From 69f4407a9a1bf898fb725ef5966a797d4c7bc9dd Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 7 Jun 2018 23:39:39 +1000
Subject: [PATCH 07/95] appveyor.yaml: run robot in powershell so exit code is
 ignored for now.

---
 appveyor.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index b6b81a3c25b..8377472b638 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -104,7 +104,8 @@ before_test:
  - mkdir testResults
 
 test_script:
- - py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot
+ - ps: |
+     py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot
  
 after_test:
  - ps: |

From 9bd9760e956bf655759db5b49f6960a13f577f35 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 7 Jun 2018 23:51:11 +1000
Subject: [PATCH 08/95] appveyor.yaml:  try again to make system tests return
 error code 0

---
 appveyor.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 8377472b638..5648cb28877 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -104,8 +104,7 @@ before_test:
  - mkdir testResults
 
 test_script:
- - ps: |
-     py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot
+ - py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot & echo done tests
  
 after_test:
  - ps: |

From 488a33e8c9beca7d32e18ced44befab98ff42486 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 7 Jun 2018 23:55:58 +1000
Subject: [PATCH 09/95] appveyor.yaml:  try again to make system tests return
 error code 0

---
 appveyor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 5648cb28877..f91f34a3d03 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -104,7 +104,7 @@ before_test:
  - mkdir testResults
 
 test_script:
- - py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot & echo done tests
+ - py -m robot --nostatusrc -d testResults -x systemTests_xunit.xml tests/system/initial.robot
  
 after_test:
  - ps: |

From f2758719483371030f92efd2f1eff2c01bcceae0 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 08:24:11 +1000
Subject: [PATCH 10/95] appveyor.yaml: fix syntax of uplodFile.

---
 appveyor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index f91f34a3d03..97545f9d253 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -109,7 +109,7 @@ test_script:
 after_test:
  - ps: |
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/xunit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests_xunit.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/xunit/$env:APPVEYOR_JOB_ID", ".\testResults\systemTests_xunit.xml")
  
 artifacts:
  - path: output\*

From 9b889f80621652ceebbe3cd24676e9e28b68923b Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 09:46:35 +1000
Subject: [PATCH 11/95] system tests: tell appVeyor they are junit formatted?

---
 appveyor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 97545f9d253..d6c1ffce204 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -109,7 +109,7 @@ test_script:
 after_test:
  - ps: |
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/xunit/$env:APPVEYOR_JOB_ID", ".\testResults\systemTests_xunit.xml")
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests_xunit.xml))
  
 artifacts:
  - path: output\*

From c3b4a9047875c99f65a1cb67cfb11c706a502a4c Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 10:39:39 +1000
Subject: [PATCH 12/95] appVeyor: still upload test results on failed builds.

---
 appveyor.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index d6c1ffce204..789f5b2de9e 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -104,13 +104,14 @@ before_test:
  - mkdir testResults
 
 test_script:
- - py -m robot --nostatusrc -d testResults -x systemTests_xunit.xml tests/system/initial.robot
- 
-after_test:
  - ps: |
+     $errorCode=0
+     py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot
+     if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests_xunit.xml))
- 
+     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+
 artifacts:
  - path: output\*
  - path: output\*\*

From 8b0e1c26307e9afe758129fdaf5f5175507a54e2 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 11:25:56 +1000
Subject: [PATCH 13/95] Upload unitTest reports to appViewer as well.

---
 appveyor.yml | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 789f5b2de9e..1a39a5a6c79 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -75,8 +75,8 @@ build_script:
      Set-AppveyorBuildVariable "sconsArgs" $sconsArgs
  - 'echo scons args: %sconsArgs%'
  - py scons.py source %sconsArgs%
- # We don't need launcher to run tests, so run the tests before launcher.
- - py scons.py tests %sconsArgs%
+ # We don't need launcher to run checkPot, so run the checkPot before launcher.
+ - py scons.py checkPot %sconsArgs%
  # The pot gets built by tests, but we don't actually need it as a build artifact.
  - del output\*.pot
  - 'echo scons output targets: %sconsOutTargets%'
@@ -101,15 +101,21 @@ before_test:
  - py -m pip install robotframework
  - py -m pip install robotremoteserver
  - py -m pip install pyautogui
+ - py -m pip install nose 
  - mkdir testResults
 
 test_script:
  - ps: |
      $errorCode=0
-     py -m robot -d testResults -x systemTests_xunit.xml tests/system/initial.robot
+     py -m nose --with-xunit --xunit-file=testResults/unitTests.xml tests/unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests_xunit.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\unitTests.xml))
+     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+     py -m robot -d testResults -x systemTests.xml tests/system/initial.robot
+     if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
+     $wc = New-Object 'System.Net.WebClient'
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
 
 artifacts:

From 9270a20c7c6329892d3715bf8582cdb8b28b55ea Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 11:55:25 +1000
Subject: [PATCH 14/95] Investigate freeze in robot test.

---
 tests/system/initial.robot | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index e8d885a9f67..cc678a5cbef 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -10,6 +10,7 @@ Library       nvdaRobotLib.py
 *** Test Cases ***
 
 Can Start and exit NVDA
-    ${nvdaResult} =  quit NVDA
-    Should Be Equal  ${nvdaResult.stdout}  Hello, world!
+    start nvda
+    quit NVDA
+
 

From 30a12566a39b324a8e592edbac657822b3a2f7f4 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 12:30:24 +1000
Subject: [PATCH 15/95] add some logging to nvdaRobotLib

---
 tests/system/nvdaRobotLib.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index e16470c92fe..e9c3ab91e4c 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -48,11 +48,17 @@ def _connectToRemoteServer(self):
 
 
 	def start_NVDA(self):
+		print "*WARN* copy plugin"
 		self.copy_in_system_test_spy()
+		print "*WARN* start NVDA process"
 		nvdaProcessHandle = self._startNVDAProcess()
+		print "*WARN* check if running"
 		process.process_should_be_running(nvdaProcessHandle)
+		print "*WARN* connect to remote server"
 		self._connectToRemoteServer()
+		print "*WARN* wait for NVDA start-up to complete"
 		self.wait_for_NVDA_startup_to_complete()
+		print "*WARN* done"
 		return nvdaProcessHandle
 
 

From 8e02fc4bd5c9e32e58c2011c10bbc792fcf6f668 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 13:15:16 +1000
Subject: [PATCH 16/95] More logging

---
 tests/system/nvdaRobotLib.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index e9c3ab91e4c..95c472d7b24 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -73,8 +73,13 @@ def quit_NVDA(self):
 			nvdaSpy.Stop Remote Server
 			Wait For Process  nvdaAlias
 		"""
+		print "*WARN* send quit NVDA keys"
 		sendKey.send_quit_NVDA_keys()
+		print "*WARN* sleep"
 		builtIn.sleep(1.0)
+		print "*WARN* send enter key"
 		sendKey.send_enter_key()
+		print "*WARN* stop remote server"
 		self.nvdaSpy.run_keyword("stop_remote_server", [], {})
+		print "*WARN* wait for NVDA process"
 		return process.wait_for_process(self.nvdaHandle)

From 3824887913b8f722e53f6e007ff2a0f4a845bb42 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 13:38:39 +1000
Subject: [PATCH 17/95] more logging again

---
 tests/system/nvdaRobotLib.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index 95c472d7b24..501d078e739 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -1,3 +1,4 @@
+import sys
 from robot.libraries.BuiltIn import BuiltIn
 from robot.libraries.OperatingSystem import OperatingSystem
 from robot.libraries.Process import Process
@@ -74,12 +75,21 @@ def quit_NVDA(self):
 			Wait For Process  nvdaAlias
 		"""
 		print "*WARN* send quit NVDA keys"
+		sys.stdout.flush()
 		sendKey.send_quit_NVDA_keys()
 		print "*WARN* sleep"
+		sys.stdout.flush()
 		builtIn.sleep(1.0)
 		print "*WARN* send enter key"
+		sys.stdout.flush()
 		sendKey.send_enter_key()
 		print "*WARN* stop remote server"
+		sys.stdout.flush()
 		self.nvdaSpy.run_keyword("stop_remote_server", [], {})
 		print "*WARN* wait for NVDA process"
-		return process.wait_for_process(self.nvdaHandle)
+		sys.stdout.flush()
+		res=process.wait_for_process(self.nvdaHandle)
+		print "*WARN* done"
+		sys.stdout.flush()
+		return res
+

From e244bcb65bf48ab16fab8242aaf0d24dad809b6f Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 13:54:24 +1000
Subject: [PATCH 18/95] sleep before connecting to server.

---
 tests/system/nvdaRobotLib.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index 501d078e739..aee66622028 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -56,6 +56,7 @@ def start_NVDA(self):
 		print "*WARN* check if running"
 		process.process_should_be_running(nvdaProcessHandle)
 		print "*WARN* connect to remote server"
+		builtIn.sleep(3.0)
 		self._connectToRemoteServer()
 		print "*WARN* wait for NVDA start-up to complete"
 		self.wait_for_NVDA_startup_to_complete()

From 1c4f7b970d6590b32034d8fc5d4304f4bed6fa0d Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 14:11:28 +1000
Subject: [PATCH 19/95] more and more logging

---
 appveyor.yml               | 2 +-
 tests/system/initial.robot | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 1a39a5a6c79..8286973e1b4 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -112,7 +112,7 @@ test_script:
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
-     py -m robot -d testResults -x systemTests.xml tests/system/initial.robot
+     py -m robot --loglevel DEBUG -d testResults -x systemTests.xml tests/system/initial.robot
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests.xml))
diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index cc678a5cbef..84eb52553a1 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -11,6 +11,7 @@ Library       nvdaRobotLib.py
 
 Can Start and exit NVDA
     start nvda
+    log to console  About to run quit NVDA
     quit NVDA
 
 

From d80c6883ee4f646f1361c1e3252e5808953f804b Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 14:36:37 +1000
Subject: [PATCH 20/95] Fix logging...

---
 tests/system/nvdaRobotLib.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index aee66622028..5d3342aa125 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -49,21 +49,14 @@ def _connectToRemoteServer(self):
 
 
 	def start_NVDA(self):
-		print "*WARN* copy plugin"
 		self.copy_in_system_test_spy()
-		print "*WARN* start NVDA process"
 		nvdaProcessHandle = self._startNVDAProcess()
-		print "*WARN* check if running"
 		process.process_should_be_running(nvdaProcessHandle)
-		print "*WARN* connect to remote server"
 		builtIn.sleep(3.0)
 		self._connectToRemoteServer()
-		print "*WARN* wait for NVDA start-up to complete"
 		self.wait_for_NVDA_startup_to_complete()
-		print "*WARN* done"
 		return nvdaProcessHandle
 
-
 	def wait_for_NVDA_startup_to_complete(self):
 		while not self.nvdaSpy.run_keyword("is_NVDA_startup_complete", [], {}):
 			builtIn.sleep(0.1)

From 5e130c0566885810c51997b2f1b940111f40657a Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 15:05:01 +1000
Subject: [PATCH 21/95] reimplement quit nvda test inside initial.robot.

---
 tests/system/initial.robot | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 84eb52553a1..4df2d8f48fe 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -11,7 +11,16 @@ Library       nvdaRobotLib.py
 
 Can Start and exit NVDA
     start nvda
-    log to console  About to run quit NVDA
-    quit NVDA
+    log to console  send quit NVDA keys 
+    send quit NVDA keys
+    log to console  sleep 1
+    sleep  1
+    log to console  send enter key
+    send enter key
+    log to console  stop remote server
+    nvdaSpy.Stop Remote Server
+    log to console  wait for process NVDA alias
+    Wait For Process  nvdaAlias
+    log to console  done
 
 

From bdb51daed0495aab8439b547ff9cc5144f88a08f Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 16:28:57 +1000
Subject: [PATCH 22/95] A try at moving everying to robot language to fix
 freezes.

---
 tests/system/initial.robot   | 45 +++++++++++++++++++++---------------
 tests/system/nvdaRobotLib.py |  2 ++
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 4df2d8f48fe..ccaa94ac224 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -1,26 +1,33 @@
 *** Settings ***
-Documentation    Basic *NVDA* and _RobotFramework_ tests
-...              Starts NVDA and exits.
-...              Run with python -m robot tests/system/initial.robot in CMD.
-Library       OperatingSystem
-Library       Process
-Library       sendKey.py
-Library       nvdaRobotLib.py
+Documentation		Basic *NVDA* and _RobotFramework_ tests
+...							Starts NVDA and exits.
+...							Run with python -m robot tests/system/initial.robot in CMD.
+Library			 OperatingSystem
+Library			 Process
+Library			 sendKey.py
+Library			 nvdaRobotLib.py
 
 *** Test Cases ***
 
 Can Start and exit NVDA
-    start nvda
-    log to console  send quit NVDA keys 
-    send quit NVDA keys
-    log to console  sleep 1
-    sleep  1
-    log to console  send enter key
-    send enter key
-    log to console  stop remote server
-    nvdaSpy.Stop Remote Server
-    log to console  wait for process NVDA alias
-    Wait For Process  nvdaAlias
-    log to console  done
+	log to console	start nvda
+	start process	pythonw nvda.pyw --debug-logging -r	 cwd=source	shell=true	alias=nvdaAlias
+	sleep	4
+	log to console	connect to remote server
+	Import Library	Remote				 WITH NAME		nvdaSpy
+	log to console	process should be running
+	process should be running
+	log to console	get process object
+	log to console	send quit NVDA keys 
+	send quit NVDA keys
+	log to console	sleep 1
+	sleep	1
+	log to console	send enter key
+	send enter key
+	log to console	stop remote server
+	nvdaSpy.Stop Remote Server
+	log to console	process.wait
+	wait for process
+	log to console	done
 
 
diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index 5d3342aa125..e5089e22c79 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -87,3 +87,5 @@ def quit_NVDA(self):
 		sys.stdout.flush()
 		return res
 
+	def wait_for_process_good(self,processObj):
+		processObj.wait()

From 12867827e2d41755e8d77979de0991c940ef6afb Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 16:41:16 +1000
Subject: [PATCH 23/95] Launch NVDA from robot with py for appveyor.

---
 tests/system/initial.robot | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index ccaa94ac224..b64d5bacf91 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -11,7 +11,7 @@ Library			 nvdaRobotLib.py
 
 Can Start and exit NVDA
 	log to console	start nvda
-	start process	pythonw nvda.pyw --debug-logging -r	 cwd=source	shell=true	alias=nvdaAlias
+	start process	py nvda.pyw --debug-logging -r	 cwd=source	shell=true	alias=nvdaAlias
 	sleep	4
 	log to console	connect to remote server
 	Import Library	Remote				 WITH NAME		nvdaSpy

From a57e95e94d64cdd97e40ab2932d53e0a9d3e8e82 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 16:45:26 +1000
Subject: [PATCH 24/95] copy the globalPlugin.

---
 tests/system/initial.robot | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index b64d5bacf91..3ba2a955e06 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -10,6 +10,8 @@ Library			 nvdaRobotLib.py
 *** Test Cases ***
 
 Can Start and exit NVDA
+	log to console	copy globalPlugin
+	Copy File	tests/system/systemTestSpy.py	source/globalPlugins/
 	log to console	start nvda
 	start process	py nvda.pyw --debug-logging -r	 cwd=source	shell=true	alias=nvdaAlias
 	sleep	4

From 5d540e702b9a01f9ada0d566a1abd0e8454f9e78 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 17:11:58 +1000
Subject: [PATCH 25/95] Temporarily start rdp for appveyor build

---
 appveyor.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/appveyor.yml b/appveyor.yml
index 8286973e1b4..9c9bcd000ff 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -20,6 +20,7 @@ environment:
 
 init:
  - ps: |
+     iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
      if ($env:APPVEYOR_REPO_TAG_NAME -and $env:APPVEYOR_REPO_TAG_NAME.StartsWith("release-")) {
       # Strip "release-" prefix.
       $version = $env:APPVEYOR_REPO_TAG_NAME.Substring(8)

From 8195ee41eb6b6ca5babedb2628ada97dba337f0e Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 17:41:00 +1000
Subject: [PATCH 26/95] bump


From 6e530b6d7b31c1d697afa052d01119a2f9a4478c Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 17:51:56 +1000
Subject: [PATCH 27/95] bump 2


From 38e696ba7e55a706436811781639bd2e4b504f8c Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 18:05:21 +1000
Subject: [PATCH 28/95] bump 3


From 2274ea6733d2afee141637b46b853262de705564 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Fri, 8 Jun 2018 18:12:26 +1000
Subject: [PATCH 29/95] bump 4


From d8cd6189a0deeb63c5b18d301ea610980da4cb1e Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Mon, 11 Jun 2018 15:01:44 +1000
Subject: [PATCH 30/95] Slightly re-write now with setup and teardown, and
 insert q for quit check  and check foreground name is Exit NVDA etc.

---
 tests/system/initial.robot   | 37 ++++++++++--------------------
 tests/system/nvdaRobotLib.py | 44 +++++++++---------------------------
 tests/system/sendKey.py      | 26 +++++++++++++--------
 3 files changed, 39 insertions(+), 68 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 3ba2a955e06..d74fd86a3cc 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -5,31 +5,18 @@ Documentation		Basic *NVDA* and _RobotFramework_ tests
 Library			 OperatingSystem
 Library			 Process
 Library			 sendKey.py
-Library			 nvdaRobotLib.py
+Library	nvdaRobotLib.py
 
-*** Test Cases ***
-
-Can Start and exit NVDA
-	log to console	copy globalPlugin
-	Copy File	tests/system/systemTestSpy.py	source/globalPlugins/
-	log to console	start nvda
-	start process	py nvda.pyw --debug-logging -r	 cwd=source	shell=true	alias=nvdaAlias
-	sleep	4
-	log to console	connect to remote server
-	Import Library	Remote				 WITH NAME		nvdaSpy
-	log to console	process should be running
-	process should be running
-	log to console	get process object
-	log to console	send quit NVDA keys 
-	send quit NVDA keys
-	log to console	sleep 1
-	sleep	1
-	log to console	send enter key
-	send enter key
-	log to console	stop remote server
-	nvdaSpy.Stop Remote Server
-	log to console	process.wait
-	wait for process
-	log to console	done
+*** Settings ***
+Test Setup	start NVDA
+Test Teardown	quit NVDA
 
+*** Test Cases ***
+Ensure NVDA runs at all
+	process should be running	nvdaAlias
 
+Ensure NVDA quits from keyboard
+	send key	insert	q
+	wait for foreground	Exit NVDA
+	send key	enter
+	wait for process	nvdaAlias	timeout=5 sec
diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index e5089e22c79..96b7b1af4bd 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -1,12 +1,10 @@
 import sys
 from robot.libraries.BuiltIn import BuiltIn
-from robot.libraries.OperatingSystem import OperatingSystem
-from robot.libraries.Process import Process
 import sendKey
 
 builtIn = BuiltIn()
-os = OperatingSystem()
-process = Process()
+os = builtIn.get_library_instance('OperatingSystem')
+process = builtIn.get_library_instance('Process')
 
 class nvdaRobotLib(object):
 
@@ -27,7 +25,7 @@ def _startNVDAProcess(self):
 		Start Process  pythonw nvda.pyw --debug-logging  cwd=source  shell=true  alias=nvdaAlias
 		"""
 		self.nvdaHandle = handle = process.start_process(
-			"pythonw nvda.pyw --debug-logging",
+			"pythonw nvda.pyw --debug-logging -r",
 			cwd='source',
 			shell=True,
 			alias='nvdaAlias'
@@ -52,7 +50,7 @@ def start_NVDA(self):
 		self.copy_in_system_test_spy()
 		nvdaProcessHandle = self._startNVDAProcess()
 		process.process_should_be_running(nvdaProcessHandle)
-		builtIn.sleep(3.0)
+		builtIn.sleep(4.0)
 		self._connectToRemoteServer()
 		self.wait_for_NVDA_startup_to_complete()
 		return nvdaProcessHandle
@@ -62,30 +60,10 @@ def wait_for_NVDA_startup_to_complete(self):
 			builtIn.sleep(0.1)
 
 	def quit_NVDA(self):
-		"""send quit NVDA keys
-			sleep  1
-			send enter key
-			nvdaSpy.Stop Remote Server
-			Wait For Process  nvdaAlias
-		"""
-		print "*WARN* send quit NVDA keys"
-		sys.stdout.flush()
-		sendKey.send_quit_NVDA_keys()
-		print "*WARN* sleep"
-		sys.stdout.flush()
-		builtIn.sleep(1.0)
-		print "*WARN* send enter key"
-		sys.stdout.flush()
-		sendKey.send_enter_key()
-		print "*WARN* stop remote server"
-		sys.stdout.flush()
-		self.nvdaSpy.run_keyword("stop_remote_server", [], {})
-		print "*WARN* wait for NVDA process"
-		sys.stdout.flush()
-		res=process.wait_for_process(self.nvdaHandle)
-		print "*WARN* done"
-		sys.stdout.flush()
-		return res
-
-	def wait_for_process_good(self,processObj):
-		processObj.wait()
+		"""send quit NVDA keys"""
+		process.run_process(
+			"pythonw nvda.pyw -q",
+			cwd='source',
+			shell=True,
+		)
+		process.wait_for_process(self.nvdaHandle)
diff --git a/tests/system/sendKey.py b/tests/system/sendKey.py
index 83102cfb1be..d3f3351ef99 100644
--- a/tests/system/sendKey.py
+++ b/tests/system/sendKey.py
@@ -1,5 +1,7 @@
+import time
 import pyautogui
 pyautogui.FAILSAFE=False
+import ctypes
 
 """
 ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(',
@@ -26,14 +28,18 @@
 'command', 'option', 'optionleft', 'optionright']
 """
 
-def send_quit_NVDA_keys():
-    """
-    Sends Caps+q
-    """
-    pyautogui.hotkey('capslock', 'q')
+def send_key(*keys):
+	pyautogui.hotkey(*keys)
 
-def send_enter_key():
-    """
-    Sends ENTER
-    """
-    pyautogui.hotkey('enter')
\ No newline at end of file
+def get_foreground_name():
+	hwnd=ctypes.windll.user32.GetForegroundWindow()
+	buf=ctypes.create_unicode_buffer(1024)
+	ctypes.windll.user32.InternalGetWindowText(hwnd,buf,1023)
+	return buf.value
+
+def wait_for_foreground(name,timeout=5):
+	endTime=time.time()+timeout
+	while get_foreground_name()!=name:
+		time.sleep(0.25)
+		if time.time()>endTime:
+			raise RuntimeError("Timeout")

From d6acae57b4a6ae3e93772791a42e305e42d1adb7 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Wed, 13 Jun 2018 08:12:14 +1000
Subject: [PATCH 31/95] nvdaRobotLib: use a specific NVDA profile dir.

---
 tests/system/nvdaRobotLib.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index 96b7b1af4bd..b2bf86fa2b7 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -1,11 +1,13 @@
 import sys
+import os
 from robot.libraries.BuiltIn import BuiltIn
 import sendKey
 
 builtIn = BuiltIn()
-os = builtIn.get_library_instance('OperatingSystem')
 process = builtIn.get_library_instance('Process')
 
+nvdaProfileDir=os.path.abspath("tests/system/nvdaProfile")
+
 class nvdaRobotLib(object):
 
 	def __init__(self):
@@ -13,19 +15,12 @@ def __init__(self):
 		self.nvdaHandle = None
 
 
-	def copy_in_system_test_spy(self):
-		"""Equiv robot text:
-		Copy File  tests/system/systemTestSpy.py  source/globalPlugins/
-		"""
-		os.copy_file("tests/system/systemTestSpy.py", "source/globalPlugins/")
-
-
 	def _startNVDAProcess(self):
 		"""Equiv robot text:
 		Start Process  pythonw nvda.pyw --debug-logging  cwd=source  shell=true  alias=nvdaAlias
 		"""
 		self.nvdaHandle = handle = process.start_process(
-			"pythonw nvda.pyw --debug-logging -r",
+			"pythonw nvda.pyw --debug-logging -r -c \"{nvdaProfileDir}\"".format(nvdaProfileDir=nvdaProfileDir),
 			cwd='source',
 			shell=True,
 			alias='nvdaAlias'
@@ -47,7 +42,6 @@ def _connectToRemoteServer(self):
 
 
 	def start_NVDA(self):
-		self.copy_in_system_test_spy()
 		nvdaProcessHandle = self._startNVDAProcess()
 		process.process_should_be_running(nvdaProcessHandle)
 		builtIn.sleep(4.0)

From 5c55400ee1d4ce9fdeab5abd5ea72a2e67e4758a Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Wed, 13 Jun 2018 08:13:07 +1000
Subject: [PATCH 32/95] NVDA Spy for robot must use a daemon thread for now to
 stop  further NVDAs crashing.

---
 .../globalPlugins/systemTestSpy.py            | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 tests/system/nvdaProfile/globalPlugins/systemTestSpy.py

diff --git a/tests/system/nvdaProfile/globalPlugins/systemTestSpy.py b/tests/system/nvdaProfile/globalPlugins/systemTestSpy.py
new file mode 100644
index 00000000000..432423096f6
--- /dev/null
+++ b/tests/system/nvdaProfile/globalPlugins/systemTestSpy.py
@@ -0,0 +1,50 @@
+import globalPluginHandler
+import signal
+import threading
+from robotremoteserver import RobotRemoteServer
+from logHandler import log
+
+class SystemTestSpy:
+	def __init__(self):
+		self._nvdaStartupComplete = False
+		from core import postNvdaStartup
+		postNvdaStartup.register(self._onNvdaStartupComplete)
+
+	def _onNvdaStartupComplete(self):
+		self._nvdaStartupComplete = True
+
+	def is_NVDA_startup_complete(self):
+		log.debug("Got startup complete action")
+		return self._nvdaStartupComplete
+
+
+class SystemTestSpyServer(object):
+	def __init__(self):
+		self._server = None
+
+	def start(self):
+		log.debug("TestSpyPlugin started")
+		server = self._server = RobotRemoteServer(
+			SystemTestSpy(),  # provides actual spy behaviour
+			port=8270,  # Could use 0 for auto port selection, which can be written out to a file / command line
+			serve=False  # we want to start this serving on another thread so as not to block.
+		)
+		log.debug("Server address: {}".format(server.server_address))
+		signal.signal(signal.SIGINT, lambda signum, frame: server.stop())
+		server_thread = threading.Thread(target=server.serve)
+		server_thread.setDaemon(True)
+		server_thread.start()
+
+	def stop(self):
+		self._server.stop()
+
+
+class GlobalPlugin(globalPluginHandler.GlobalPlugin):
+
+	def __init__(self):
+		super(GlobalPlugin, self).__init__()
+		self._testSpy = SystemTestSpyServer()
+		self._testSpy.start()
+
+	__gestures = {
+	}
\ No newline at end of file

From 06aa06b1a2577e48e9cf120fc7738dbcb9dcb87b Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Wed, 13 Jun 2018 08:15:47 +1000
Subject: [PATCH 33/95] Robot tests set NVDA not to show the welcome dialog

---
 tests/system/nvdaProfile/nvda.ini |  3 ++
 tests/system/systemTestSpy.py     | 49 -------------------------------
 2 files changed, 3 insertions(+), 49 deletions(-)
 create mode 100644 tests/system/nvdaProfile/nvda.ini
 delete mode 100644 tests/system/systemTestSpy.py

diff --git a/tests/system/nvdaProfile/nvda.ini b/tests/system/nvdaProfile/nvda.ini
new file mode 100644
index 00000000000..e62f2942cb4
--- /dev/null
+++ b/tests/system/nvdaProfile/nvda.ini
@@ -0,0 +1,3 @@
+schemaVersion = 2
+[general]
+	showWelcomeDialogAtStartup = False
diff --git a/tests/system/systemTestSpy.py b/tests/system/systemTestSpy.py
deleted file mode 100644
index 0d5e9db0745..00000000000
--- a/tests/system/systemTestSpy.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import globalPluginHandler
-import signal
-import threading
-from robotremoteserver import RobotRemoteServer
-from logHandler import log
-
-class SystemTestSpy:
-	def __init__(self):
-		self._nvdaStartupComplete = False
-		from core import postNvdaStartup
-		postNvdaStartup.register(self._onNvdaStartupComplete)
-
-	def _onNvdaStartupComplete(self):
-		self._nvdaStartupComplete = True
-
-	def is_NVDA_startup_complete(self):
-		log.debug("Got startup complete action")
-		return self._nvdaStartupComplete
-
-
-class SystemTestSpyServer(object):
-	def __init__(self):
-		self._server = None
-
-	def start(self):
-		log.debug("TestSpyPlugin started")
-		server = self._server = RobotRemoteServer(
-			SystemTestSpy(),  # provides actual spy behaviour
-			port=8270,  # Could use 0 for auto port selection, which can be written out to a file / command line
-			serve=False  # we want to start this serving on another thread so as not to block.
-		)
-		log.debug("Server address: {}".format(server.server_address))
-		signal.signal(signal.SIGINT, lambda signum, frame: server.stop())
-		server_thread = threading.Thread(target=server.serve)
-		server_thread.start()
-
-	def stop(self):
-		self._server.stop()
-
-
-class GlobalPlugin(globalPluginHandler.GlobalPlugin):
-
-	def __init__(self):
-		super(GlobalPlugin, self).__init__()
-		self._testSpy = SystemTestSpyServer()
-		self._testSpy.start()
-
-	__gestures = {
-	}
\ No newline at end of file

From ce75aa0fe73b4a068052d02869b81bb9f165db7b Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Thu, 14 Jun 2018 01:01:32 +0000
Subject: [PATCH 34/95] allow asserting on last speech

---
 source/speech.py              |  7 ++++++-
 tests/system/initial.robot    |  2 +-
 tests/system/nvdaRobotLib.py  |  5 +++++
 tests/system/systemTestSpy.py | 15 +++++++++++++++
 4 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/source/speech.py b/source/speech.py
index 8acd60e2071..82169f028cc 100755
--- a/source/speech.py
+++ b/source/speech.py
@@ -27,6 +27,10 @@
 import speechDictHandler
 import characterProcessing
 import languageHandler
+import extensionPoints
+
+# inform those who want to know that there is new speech
+preSpeech = extensionPoints.Action()
 
 speechMode_off=0
 speechMode_beeps=1
@@ -493,10 +497,11 @@ def speak(speechSequence,symbolLevel=None):
 	"""
 	if not speechSequence: #Pointless - nothing to speak 
 		return
+	preSpeech.notify(speechSequence=speechSequence)
 	import speechViewer
 	if speechViewer.isActive:
 		for item in speechSequence:
-			if isinstance(item,basestring):
+			if isinstance(item, basestring):
 				speechViewer.appendText(item)
 	global beenCanceled, curWordChars
 	curWordChars=[]
diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 863f6b792dc..6b4546c0342 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -12,5 +12,5 @@ Library       nvdaRobotLib.py
 Can Start and exit NVDA
     ${nvdaProcessHandle} =  start NVDA
     ${nvdaResult} =  quit NVDA
-    Should Be Equal  ${nvdaResult.stdout}  Hello, world!
+    assert last speech  "Welcome to NVDA"
 
diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index e16470c92fe..1705f8620f8 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -60,6 +60,7 @@ def wait_for_NVDA_startup_to_complete(self):
 		while not self.nvdaSpy.run_keyword("is_NVDA_startup_complete", [], {}):
 			builtIn.sleep(0.1)
 
+
 	def quit_NVDA(self):
 		"""send quit NVDA keys
 			sleep  1
@@ -72,3 +73,7 @@ def quit_NVDA(self):
 		sendKey.send_enter_key()
 		self.nvdaSpy.run_keyword("stop_remote_server", [], {})
 		return process.wait_for_process(self.nvdaHandle)
+
+	def assert_last_speech(self, expectedSpeech):
+		actualLastSpeech = self.nvdaSpy.run_keyword("get_last_speech", [], {})
+		builtIn.should_be_equal_as_strings(actualLastSpeech, expectedSpeech)
diff --git a/tests/system/systemTestSpy.py b/tests/system/systemTestSpy.py
index 0d5e9db0745..eb3bdcff0ad 100644
--- a/tests/system/systemTestSpy.py
+++ b/tests/system/systemTestSpy.py
@@ -8,15 +8,30 @@ class SystemTestSpy:
 	def __init__(self):
 		self._nvdaStartupComplete = False
 		from core import postNvdaStartup
+		from speech import preSpeech
 		postNvdaStartup.register(self._onNvdaStartupComplete)
+		preSpeech.register(self._onNvdaSpeech)
+		self._nvdaSpeech = [
+			[""],  # initialise with an empty string, this allows for access via [-1]. This is equiv to no speech.
+		]
 
 	def _onNvdaStartupComplete(self):
 		self._nvdaStartupComplete = True
 
+	def _onNvdaSpeech(self, speechSequence=None):
+		if not speechSequence: return
+		self._nvdaSpeech.append(speechSequence)
+
 	def is_NVDA_startup_complete(self):
 		log.debug("Got startup complete action")
 		return self._nvdaStartupComplete
 
+	def get_last_speech(self):
+		baseStrings = [s.strip() for s in self._nvdaSpeech[-1] if isinstance(s, basestring)]
+		lastSpeech = ' '.join(baseStrings)
+		log.debug("last speech: {}".format(lastSpeech))
+		return lastSpeech
+
 
 class SystemTestSpyServer(object):
 	def __init__(self):

From 735991aa8563c82e3afa04f1d1b2644deee63a56 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Tue, 19 Jun 2018 04:39:19 +0000
Subject: [PATCH 35/95] Remove extra tab characters

---
 tests/system/initial.robot | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 410d3205a37..87429cb1dd2 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -1,10 +1,10 @@
 *** Settings ***
-Documentation		Basic *NVDA* and _RobotFramework_ tests
-...							Starts NVDA and exits.
-...							Run with python -m robot tests/system/initial.robot in CMD.
-Library			 OperatingSystem
-Library			 Process
-Library			 sendKey.py
+Documentation	Basic *NVDA* and _RobotFramework_ tests
+...	Starts NVDA and exits.
+...	Run with python -m robot tests/system/initial.robot in CMD.
+Library	OperatingSystem
+Library	Process
+Library	sendKey.py
 Library	nvdaRobotLib.py
 
 *** Settings ***
@@ -22,4 +22,4 @@ Ensure NVDA quits from keyboard
 	wait for process	nvdaAlias	timeout=5 sec
 
 Can Start and exit NVDA
-	assert last speech  "Welcome to NVDA"
\ No newline at end of file
+	assert last speech	"Welcome to NVDA"
\ No newline at end of file

From 662eabbd3fdba628113d9d2dd1141d57b73bd81a Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 02:16:53 +0000
Subject: [PATCH 36/95] Copy in the testSpy globalPlugin for each test.

---
 source/nvda.pyw                               |  2 ++
 tests/system/nvdaRobotLib.py                  | 32 +++++++++++++++----
 .../globalPlugins => }/systemTestSpy.py       |  9 +++---
 3 files changed, 32 insertions(+), 11 deletions(-)
 rename tests/system/{nvdaProfile/globalPlugins => }/systemTestSpy.py (86%)

diff --git a/source/nvda.pyw b/source/nvda.pyw
index d0d78e87e16..d023322cb4c 100755
--- a/source/nvda.pyw
+++ b/source/nvda.pyw
@@ -194,6 +194,8 @@ if globalVars.appArgs.debugLogging:
 	logLevel=log.DEBUG
 logHandler.initialize()
 logHandler.log.setLevel(logLevel)
+if logLevel is log.DEBUG:
+	log.debug("Provided arguments: {}".format(sys.argv[1:]))
 
 log.info("Starting NVDA")
 log.debug("Debug level logging enabled")
diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index 2fcb0ff3802..acd061f9c7b 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -1,12 +1,18 @@
-import sys
 import os
-from robot.libraries.BuiltIn import BuiltIn
-import sendKey
+import sys
+from timeit import default_timer as timer
 
+from robot.libraries.BuiltIn import BuiltIn
 builtIn = BuiltIn()
 process = builtIn.get_library_instance('Process')
+opSys = builtIn.get_library_instance('OperatingSystem')
 
-nvdaProfileDir=os.path.abspath("tests/system/nvdaProfile")
+systemTestSpyFileName = "systemTestSpy.py"
+systemTestSourceDir = os.path.abspath("tests/system")
+nvdaProfileDir = os.path.join(systemTestSourceDir, "nvdaProfile")
+systemTestSpySource = os.path.join(systemTestSourceDir, systemTestSpyFileName)
+systemTestSpyInstallDir = os.path.join(nvdaProfileDir, "globalPlugins")
+systemTestSpyInstalled = os.path.join(systemTestSpyInstallDir, systemTestSpyFileName)
 
 class nvdaRobotLib(object):
 
@@ -14,6 +20,14 @@ def __init__(self):
 		self.nvdaSpy = None
 		self.nvdaHandle = None
 
+	def copy_in_system_test_spy(self):
+		"""Equiv robot text:
+		Copy File  tests/system/systemTestSpy.py  {nvdaProfile}/globalPlugins/
+		"""
+		opSys.copy_file(systemTestSpySource, systemTestSpyInstallDir)
+
+	def remove_system_test_spy(self):
+		opSys.remove_file(systemTestSpyInstalled)
 
 	def _startNVDAProcess(self):
 		"""Equiv robot text:
@@ -27,7 +41,6 @@ def _startNVDAProcess(self):
 		)
 		return handle
 
-
 	def _connectToRemoteServer(self):
 		"""Equiv robot text:
 		Import Library  Remote         WITH NAME    nvdaSpy
@@ -42,6 +55,7 @@ def _connectToRemoteServer(self):
 
 
 	def start_NVDA(self):
+		self.copy_in_system_test_spy()
 		nvdaProcessHandle = self._startNVDAProcess()
 		process.process_should_be_running(nvdaProcessHandle)
 		builtIn.sleep(4.0)
@@ -55,9 +69,13 @@ def wait_for_NVDA_startup_to_complete(self):
 
 
 	def quit_NVDA(self):
-		"""send quit NVDA keys"""
+		try:
+			self.nvdaSpy.run_keyword("stop_remote_server", [], {})
+		except RuntimeError:
+			pass  # if the test manually exits, then we are unable to run this keyword.
+		self.remove_system_test_spy()
 		process.run_process(
-			"pythonw nvda.pyw -q",
+			"pythonw nvda.pyw -q --disable-addons",
 			cwd='source',
 			shell=True,
 		)
diff --git a/tests/system/nvdaProfile/globalPlugins/systemTestSpy.py b/tests/system/systemTestSpy.py
similarity index 86%
rename from tests/system/nvdaProfile/globalPlugins/systemTestSpy.py
rename to tests/system/systemTestSpy.py
index e022cc7aabe..6d99035cd02 100644
--- a/tests/system/nvdaProfile/globalPlugins/systemTestSpy.py
+++ b/tests/system/systemTestSpy.py
@@ -41,19 +41,16 @@ def start(self):
 		log.debug("TestSpyPlugin started")
 		server = self._server = RobotRemoteServer(
 			SystemTestSpy(),  # provides actual spy behaviour
-			port=8270,  # Could use 0 for auto port selection, which can be written out to a file / command line
+			port=8270,  # default:8270 is `registered by IANA` for remote server usage. Two ASCII values, RF.
 			serve=False  # we want to start this serving on another thread so as not to block.
 		)
 		log.debug("Server address: {}".format(server.server_address))
-		signal.signal(signal.SIGINT, lambda signum, frame: server.stop())
 		server_thread = threading.Thread(target=server.serve)
-		server_thread.setDaemon(True)
 		server_thread.start()
 
 	def stop(self):
 		self._server.stop()
 
-
 class GlobalPlugin(globalPluginHandler.GlobalPlugin):
 
 	def __init__(self):
@@ -61,5 +58,9 @@ def __init__(self):
 		self._testSpy = SystemTestSpyServer()
 		self._testSpy.start()
 
+	def terminate(self):
+		log.debug("Terminating the systemTestSpy")
+		self._testSpy.stop()
+
 	__gestures = {
 	}
\ No newline at end of file

From 08c869768307b7847f1d8187d30ffe2e5788d810 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 02:18:07 +0000
Subject: [PATCH 37/95] Remove sleep

The sleep was required because importing the remote library suceeds
regardless of whether it is possible to execute a keyword. We now test
the connection before we load the library.
---
 tests/system/nvdaRobotLib.py | 37 +++++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index acd061f9c7b..37909598d36 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -45,20 +45,43 @@ def _connectToRemoteServer(self):
 		"""Equiv robot text:
 		Import Library  Remote         WITH NAME    nvdaSpy
 		"""
-		builtIn.import_library(
-				"Remote",
-				'http://127.0.0.1:8270',
+		port = 8270  # default:8270 is `registered by IANA` for remote server usage. Two ASCII values, RF.
+		uri = 'http://127.0.0.1:{}'.format(port)
+		spyAlias = "nvdaSpy"
+
+		startTime = timer()
+		giveUpAfter = 10  # seconds
+		intervalBetweenTries = 0.1  # seconds
+		lastRunTime = startTime - intervalBetweenTries+1  # ensure we start trying immediately
+		canConnect=False
+		from robotremoteserver import test_remote_server
+		while not canConnect and (timer() - startTime) < giveUpAfter:
+			if (timer() - lastRunTime) > intervalBetweenTries:
+				lastRunTime = timer()
+
+				# Importing the 'Remote' library always succeeds, even when a connection can not be made.
+				# If that happens, then some 'Remote' keyword will fail at some later point.
+				# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.
+				canConnect = test_remote_server(uri)
+
+		if not canConnect:
+			raise RuntimeError("Unable to connect to nvdaSpy")
+		else:
+			builtIn.import_library(
+				"Remote",  # name of library to import
+				# Arguments to construct the library instance:
+				"uri={}".format(uri),
+				"timeout=2",  # seconds
+				# Set an alias for the imported library instance
 				"WITH NAME",
-				"nvdaSpy"
+				"nvdaSpy",
 			)
-		self.nvdaSpy = builtIn.get_library_instance("nvdaSpy")
-
+			self.nvdaSpy = builtIn.get_library_instance(spyAlias)
 
 	def start_NVDA(self):
 		self.copy_in_system_test_spy()
 		nvdaProcessHandle = self._startNVDAProcess()
 		process.process_should_be_running(nvdaProcessHandle)
-		builtIn.sleep(4.0)
 		self._connectToRemoteServer()
 		self.wait_for_NVDA_startup_to_complete()
 		return nvdaProcessHandle

From 89816971a52e7195e30cce18611d334dab8487bb Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 02:43:06 +0000
Subject: [PATCH 38/95] Some cleanup of connection/disconnection code

---
 tests/system/nvdaRobotLib.py | 51 +++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index 37909598d36..52241b39a67 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -1,12 +1,18 @@
 import os
 import sys
 from timeit import default_timer as timer
-
+from robotremoteserver import test_remote_server, stop_remote_server
 from robot.libraries.BuiltIn import BuiltIn
+
+
 builtIn = BuiltIn()
 process = builtIn.get_library_instance('Process')
 opSys = builtIn.get_library_instance('OperatingSystem')
 
+spyServerPort = 8270  # is `registered by IANA` for remote server usage. Two ASCII values:'RF'
+spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
+spyAlias = "nvdaSpy"
+
 systemTestSpyFileName = "systemTestSpy.py"
 systemTestSourceDir = os.path.abspath("tests/system")
 nvdaProfileDir = os.path.join(systemTestSourceDir, "nvdaProfile")
@@ -45,38 +51,33 @@ def _connectToRemoteServer(self):
 		"""Equiv robot text:
 		Import Library  Remote         WITH NAME    nvdaSpy
 		"""
-		port = 8270  # default:8270 is `registered by IANA` for remote server usage. Two ASCII values, RF.
-		uri = 'http://127.0.0.1:{}'.format(port)
-		spyAlias = "nvdaSpy"
-
 		startTime = timer()
 		giveUpAfter = 10  # seconds
 		intervalBetweenTries = 0.1  # seconds
 		lastRunTime = startTime - intervalBetweenTries+1  # ensure we start trying immediately
-		canConnect=False
-		from robotremoteserver import test_remote_server
-		while not canConnect and (timer() - startTime) < giveUpAfter:
+
+		while (timer() - startTime) < giveUpAfter:
 			if (timer() - lastRunTime) > intervalBetweenTries:
 				lastRunTime = timer()
 
 				# Importing the 'Remote' library always succeeds, even when a connection can not be made.
 				# If that happens, then some 'Remote' keyword will fail at some later point.
 				# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.
-				canConnect = test_remote_server(uri)
-
-		if not canConnect:
-			raise RuntimeError("Unable to connect to nvdaSpy")
+				if test_remote_server(spyServerURI, log=False):
+					break
 		else:
-			builtIn.import_library(
-				"Remote",  # name of library to import
-				# Arguments to construct the library instance:
-				"uri={}".format(uri),
-				"timeout=2",  # seconds
-				# Set an alias for the imported library instance
-				"WITH NAME",
-				"nvdaSpy",
-			)
-			self.nvdaSpy = builtIn.get_library_instance(spyAlias)
+			raise RuntimeError("Unable to connect to nvdaSpy")
+
+		builtIn.import_library(
+			"Remote",  # name of library to import
+			# Arguments to construct the library instance:
+			"uri={}".format(spyServerURI),
+			"timeout=2",  # seconds
+			# Set an alias for the imported library instance
+			"WITH NAME",
+			"nvdaSpy",
+		)
+		self.nvdaSpy = builtIn.get_library_instance(spyAlias)
 
 	def start_NVDA(self):
 		self.copy_in_system_test_spy()
@@ -90,12 +91,8 @@ def wait_for_NVDA_startup_to_complete(self):
 		while not self.nvdaSpy.run_keyword("is_NVDA_startup_complete", [], {}):
 			builtIn.sleep(0.1)
 
-
 	def quit_NVDA(self):
-		try:
-			self.nvdaSpy.run_keyword("stop_remote_server", [], {})
-		except RuntimeError:
-			pass  # if the test manually exits, then we are unable to run this keyword.
+		stop_remote_server(spyServerURI, log=False)
 		self.remove_system_test_spy()
 		process.run_process(
 			"pythonw nvda.pyw -q --disable-addons",

From 669da0a8e277b4f9b170eb702cb285a05c2ad710 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 03:04:02 +0000
Subject: [PATCH 39/95] Extract code that can be reused and add better
 documentation

---
 tests/system/nvdaRobotLib.py | 59 +++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/tests/system/nvdaRobotLib.py b/tests/system/nvdaRobotLib.py
index 52241b39a67..4f10ce6f85e 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/nvdaRobotLib.py
@@ -1,5 +1,4 @@
 import os
-import sys
 from timeit import default_timer as timer
 from robotremoteserver import test_remote_server, stop_remote_server
 from robot.libraries.BuiltIn import BuiltIn
@@ -27,17 +26,14 @@ def __init__(self):
 		self.nvdaHandle = None
 
 	def copy_in_system_test_spy(self):
-		"""Equiv robot text:
-		Copy File  tests/system/systemTestSpy.py  {nvdaProfile}/globalPlugins/
-		"""
 		opSys.copy_file(systemTestSpySource, systemTestSpyInstallDir)
 
 	def remove_system_test_spy(self):
 		opSys.remove_file(systemTestSpyInstalled)
 
 	def _startNVDAProcess(self):
-		"""Equiv robot text:
-		Start Process  pythonw nvda.pyw --debug-logging  cwd=source  shell=true  alias=nvdaAlias
+		"""Start NVDA.
+		Use debug logging, replacing any current instance, using the system test profile directory
 		"""
 		self.nvdaHandle = handle = process.start_process(
 			"pythonw nvda.pyw --debug-logging -r -c \"{nvdaProfileDir}\"".format(nvdaProfileDir=nvdaProfileDir),
@@ -47,26 +43,36 @@ def _startNVDAProcess(self):
 		)
 		return handle
 
-	def _connectToRemoteServer(self):
-		"""Equiv robot text:
-		Import Library  Remote         WITH NAME    nvdaSpy
-		"""
+	def _blockUntilReturnsTrue(self, giveUpAfterSeconds, intervalBetweenSeconds, errorMessage, func):
 		startTime = timer()
-		giveUpAfter = 10  # seconds
-		intervalBetweenTries = 0.1  # seconds
-		lastRunTime = startTime - intervalBetweenTries+1  # ensure we start trying immediately
-
-		while (timer() - startTime) < giveUpAfter:
-			if (timer() - lastRunTime) > intervalBetweenTries:
+		lastRunTime = startTime - intervalBetweenSeconds+1  # ensure we start trying immediately
+		while (timer() - startTime) < giveUpAfterSeconds:
+			if (timer() - lastRunTime) > intervalBetweenSeconds:
 				lastRunTime = timer()
-
-				# Importing the 'Remote' library always succeeds, even when a connection can not be made.
-				# If that happens, then some 'Remote' keyword will fail at some later point.
-				# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.
-				if test_remote_server(spyServerURI, log=False):
+				if func():
 					break
 		else:
-			raise RuntimeError("Unable to connect to nvdaSpy")
+			raise RuntimeError(errorMessage)
+
+	def _connectToRemoteServer(self):
+		"""Connects to the nvdaSpyServer
+		Because we do not know how far through the startup NVDA is, we have to poll
+		to check that the server is available. Importing the library immediately seems
+		to succeed, but then calling a keyword later fails with RuntimeError:
+			"Connection to remote server broken: [Errno 10061]
+				No connection could be made because the target machine actively refused it"
+		Instead we wait until the remote server is available before importing the library and continuing.
+		"""
+
+		# Importing the 'Remote' library always succeeds, even when a connection can not be made.
+		# If that happens, then some 'Remote' keyword will fail at some later point.
+		# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.
+		self._blockUntilReturnsTrue(
+			giveUpAfterSeconds=10,
+			intervalBetweenSeconds=0.1,
+			errorMessage="Unable to connect to nvdaSpy",
+			func=lambda: test_remote_server(spyServerURI, log=False)
+		)
 
 		builtIn.import_library(
 			"Remote",  # name of library to import
@@ -88,11 +94,16 @@ def start_NVDA(self):
 		return nvdaProcessHandle
 
 	def wait_for_NVDA_startup_to_complete(self):
-		while not self.nvdaSpy.run_keyword("is_NVDA_startup_complete", [], {}):
-			builtIn.sleep(0.1)
+		self._blockUntilReturnsTrue(
+			giveUpAfterSeconds=10,
+			intervalBetweenSeconds=0.1,
+			errorMessage="Unable to connect to nvdaSpy",
+			func=lambda: self.nvdaSpy.run_keyword("is_NVDA_startup_complete", [], {})
+		)
 
 	def quit_NVDA(self):
 		stop_remote_server(spyServerURI, log=False)
+		# remove the spy so that if nvda is run manually against this config it does not interfere.
 		self.remove_system_test_spy()
 		process.run_process(
 			"pythonw nvda.pyw -q --disable-addons",

From 2bc38042b6c69c0886e34bc0cc3f24d089d7a62e Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 05:34:42 +0000
Subject: [PATCH 40/95] cleanup systemTest dir

Allow use of different settings files and some cleanup of system test directories
---
 tests/system/.gitignore                       |  1 +
 tests/system/initial.robot                    | 14 +++++-----
 tests/system/{ => libraries}/nvdaRobotLib.py  | 27 ++++++++++++-------
 tests/system/{ => libraries}/sendKey.py       |  0
 tests/system/{ => libraries}/systemTestSpy.py |  0
 .../standard-doShowWelcomeDialog.ini          |  3 +++
 .../standard-dontShowWelcomeDialog.ini}       |  0
 tests/system/readme.md                        | 27 +++++++++++++++++++
 8 files changed, 56 insertions(+), 16 deletions(-)
 create mode 100644 tests/system/.gitignore
 rename tests/system/{ => libraries}/nvdaRobotLib.py (80%)
 rename tests/system/{ => libraries}/sendKey.py (100%)
 rename tests/system/{ => libraries}/systemTestSpy.py (100%)
 create mode 100644 tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
 rename tests/system/{nvdaProfile/nvda.ini => nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini} (100%)
 create mode 100644 tests/system/readme.md

diff --git a/tests/system/.gitignore b/tests/system/.gitignore
new file mode 100644
index 00000000000..cec98301fd4
--- /dev/null
+++ b/tests/system/.gitignore
@@ -0,0 +1 @@
+nvdaProfile/
diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 87429cb1dd2..49c9998fd7a 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -1,14 +1,13 @@
 *** Settings ***
-Documentation	Basic *NVDA* and _RobotFramework_ tests
-...	Starts NVDA and exits.
-...	Run with python -m robot tests/system/initial.robot in CMD.
+Documentation	Basic start and exit tests
+
 Library	OperatingSystem
 Library	Process
-Library	sendKey.py
-Library	nvdaRobotLib.py
+Library	libraries/sendKey.py
+Library	libraries/nvdaRobotLib.py
 
 *** Settings ***
-Test Setup	start NVDA
+Test Setup	start NVDA	standard-dontShowWelcomeDialog.ini
 Test Teardown	quit NVDA
 
 *** Test Cases ***
@@ -21,5 +20,6 @@ Ensure NVDA quits from keyboard
 	send key	enter
 	wait for process	nvdaAlias	timeout=5 sec
 
-Can Start and exit NVDA
+Can read the welcome dialog
+	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
 	assert last speech	"Welcome to NVDA"
\ No newline at end of file
diff --git a/tests/system/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
similarity index 80%
rename from tests/system/nvdaRobotLib.py
rename to tests/system/libraries/nvdaRobotLib.py
index 4f10ce6f85e..78531c63aca 100644
--- a/tests/system/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -14,9 +14,10 @@
 
 systemTestSpyFileName = "systemTestSpy.py"
 systemTestSourceDir = os.path.abspath("tests/system")
-nvdaProfileDir = os.path.join(systemTestSourceDir, "nvdaProfile")
-systemTestSpySource = os.path.join(systemTestSourceDir, systemTestSpyFileName)
-systemTestSpyInstallDir = os.path.join(nvdaProfileDir, "globalPlugins")
+nvdaProfileWorkingDir = os.path.join(systemTestSourceDir, "nvdaProfile")
+nvdaSettingsSourceDir = os.path.join(systemTestSourceDir, "nvdaSettingsFiles")
+systemTestSpySource = os.path.join(systemTestSourceDir, "libraries", systemTestSpyFileName)
+systemTestSpyInstallDir = os.path.join(nvdaProfileWorkingDir, "globalPlugins")
 systemTestSpyInstalled = os.path.join(systemTestSpyInstallDir, systemTestSpyFileName)
 
 class nvdaRobotLib(object):
@@ -25,18 +26,26 @@ def __init__(self):
 		self.nvdaSpy = None
 		self.nvdaHandle = None
 
-	def copy_in_system_test_spy(self):
+	def setup_nvda_profile(self, settingsFileName):
 		opSys.copy_file(systemTestSpySource, systemTestSpyInstallDir)
+		opSys.copy_file(
+			os.path.join(nvdaSettingsSourceDir, settingsFileName),
+			os.path.join(nvdaProfileWorkingDir, "nvda.ini")
+		)
 
-	def remove_system_test_spy(self):
+	def teardown_nvda_profile(self):
 		opSys.remove_file(systemTestSpyInstalled)
+		opSys.remove_file(systemTestSpyInstalled+"c")  # also remove the .pyc
+		opSys.remove_file(
+			os.path.join(nvdaProfileWorkingDir, "nvda.ini")
+		)
 
 	def _startNVDAProcess(self):
 		"""Start NVDA.
 		Use debug logging, replacing any current instance, using the system test profile directory
 		"""
 		self.nvdaHandle = handle = process.start_process(
-			"pythonw nvda.pyw --debug-logging -r -c \"{nvdaProfileDir}\"".format(nvdaProfileDir=nvdaProfileDir),
+			"pythonw nvda.pyw --debug-logging -r -c \"{nvdaProfileDir}\"".format(nvdaProfileDir=nvdaProfileWorkingDir),
 			cwd='source',
 			shell=True,
 			alias='nvdaAlias'
@@ -85,8 +94,8 @@ def _connectToRemoteServer(self):
 		)
 		self.nvdaSpy = builtIn.get_library_instance(spyAlias)
 
-	def start_NVDA(self):
-		self.copy_in_system_test_spy()
+	def start_NVDA(self, settingsFileName):
+		self.setup_nvda_profile(settingsFileName)
 		nvdaProcessHandle = self._startNVDAProcess()
 		process.process_should_be_running(nvdaProcessHandle)
 		self._connectToRemoteServer()
@@ -104,7 +113,7 @@ def wait_for_NVDA_startup_to_complete(self):
 	def quit_NVDA(self):
 		stop_remote_server(spyServerURI, log=False)
 		# remove the spy so that if nvda is run manually against this config it does not interfere.
-		self.remove_system_test_spy()
+		self.teardown_nvda_profile()
 		process.run_process(
 			"pythonw nvda.pyw -q --disable-addons",
 			cwd='source',
diff --git a/tests/system/sendKey.py b/tests/system/libraries/sendKey.py
similarity index 100%
rename from tests/system/sendKey.py
rename to tests/system/libraries/sendKey.py
diff --git a/tests/system/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
similarity index 100%
rename from tests/system/systemTestSpy.py
rename to tests/system/libraries/systemTestSpy.py
diff --git a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
new file mode 100644
index 00000000000..386aa4d5d2a
--- /dev/null
+++ b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
@@ -0,0 +1,3 @@
+schemaVersion = 2
+[general]
+	showWelcomeDialogAtStartup = True
diff --git a/tests/system/nvdaProfile/nvda.ini b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
similarity index 100%
rename from tests/system/nvdaProfile/nvda.ini
rename to tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
diff --git a/tests/system/readme.md b/tests/system/readme.md
new file mode 100644
index 00000000000..531c49c4aff
--- /dev/null
+++ b/tests/system/readme.md
@@ -0,0 +1,27 @@
+## NVDA system tests
+
+### Dependencies
+
+The system tests depend on the following:
+
+- Robot Framework
+- Robot Remote Server
+- PyAutoGui
+
+Which can be installed with `pip`:
+
+```
+pip install robotframework
+pip install robotremoteserver
+pip install pyautogui
+```
+
+### Running the tests
+
+These tests should be run from the windows command prompt (cmd.exe).
+
+From the root directory of your NVDA repository, run:
+
+```
+python -m robot tests/system/ in CMD.
+```
\ No newline at end of file

From 6d2032a985bffad489229ed486b3ff47dd0b7aab Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 06:54:19 +0000
Subject: [PATCH 41/95] Assert text on welcome dialog passes

---
 tests/system/initial.robot              |  5 +++--
 tests/system/libraries/nvdaRobotLib.py  | 17 +++++++++++++++--
 tests/system/libraries/systemTestSpy.py | 24 ++++++++++++++++--------
 tests/system/variables.py               | 13 +++++++++++++
 4 files changed, 47 insertions(+), 12 deletions(-)
 create mode 100644 tests/system/variables.py

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 49c9998fd7a..14c76487893 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -6,10 +6,11 @@ Library	Process
 Library	libraries/sendKey.py
 Library	libraries/nvdaRobotLib.py
 
-*** Settings ***
 Test Setup	start NVDA	standard-dontShowWelcomeDialog.ini
 Test Teardown	quit NVDA
 
+Variables	variables.py
+
 *** Test Cases ***
 Ensure NVDA runs at all
 	process should be running	nvdaAlias
@@ -22,4 +23,4 @@ Ensure NVDA quits from keyboard
 
 Can read the welcome dialog
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
-	assert last speech	"Welcome to NVDA"
\ No newline at end of file
+	assert all speech	${WELCOME_DIALOG_TEXT}
\ No newline at end of file
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 78531c63aca..2a259c4e713 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -94,6 +94,11 @@ def _connectToRemoteServer(self):
 		)
 		self.nvdaSpy = builtIn.get_library_instance(spyAlias)
 
+	def _runNvdaSpyKeyword(self, keyword, *args, **kwargs):
+		if not args: args = []
+		if not kwargs: kwargs = {}
+		return self.nvdaSpy.run_keyword(keyword, args, kwargs)
+
 	def start_NVDA(self, settingsFileName):
 		self.setup_nvda_profile(settingsFileName)
 		nvdaProcessHandle = self._startNVDAProcess()
@@ -107,7 +112,7 @@ def wait_for_NVDA_startup_to_complete(self):
 			giveUpAfterSeconds=10,
 			intervalBetweenSeconds=0.1,
 			errorMessage="Unable to connect to nvdaSpy",
-			func=lambda: self.nvdaSpy.run_keyword("is_NVDA_startup_complete", [], {})
+			func=lambda: self._runNvdaSpyKeyword("is_NVDA_startup_complete")
 		)
 
 	def quit_NVDA(self):
@@ -122,5 +127,13 @@ def quit_NVDA(self):
 		process.wait_for_process(self.nvdaHandle)
 
 	def assert_last_speech(self, expectedSpeech):
-		actualLastSpeech = self.nvdaSpy.run_keyword("get_last_speech", [], {})
+		actualLastSpeech = self._runNvdaSpyKeyword("get_last_speech")
 		builtIn.should_be_equal_as_strings(actualLastSpeech, expectedSpeech)
+
+	def assert_all_speech(self, expectedSpeech):
+			actualSpeech = self._runNvdaSpyKeyword("get_all_speech")
+			builtIn.should_be_equal_as_strings(
+				actualSpeech,
+				expectedSpeech,
+				msg="Actual speech != to expected speech.",
+			)
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index 6d99035cd02..942fa571cde 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -11,26 +11,34 @@ def __init__(self):
 		from speech import preSpeech
 		postNvdaStartup.register(self._onNvdaStartupComplete)
 		preSpeech.register(self._onNvdaSpeech)
-		self._nvdaSpeech = [
-			[""],  # initialise with an empty string, this allows for access via [-1]. This is equiv to no speech.
-		]
+		self.clear_speech_cache()  # set self._nvdaSpeech
 
 	def _onNvdaStartupComplete(self):
 		self._nvdaStartupComplete = True
+		self.clear_speech_cache()
 
 	def _onNvdaSpeech(self, speechSequence=None):
 		if not speechSequence: return
 		self._nvdaSpeech.append(speechSequence)
 
 	def is_NVDA_startup_complete(self):
-		log.debug("Got startup complete action")
 		return self._nvdaStartupComplete
 
+	def _getJoinedBaseStringsFromCommands(self, speechCommandArray):
+		baseStrings = [c.strip() for c in speechCommandArray if isinstance(c, basestring)]
+		return ' '.join(baseStrings).replace("  ", "\n").strip()
+
 	def get_last_speech(self):
-		baseStrings = [s.strip() for s in self._nvdaSpeech[-1] if isinstance(s, basestring)]
-		lastSpeech = ' '.join(baseStrings)
-		log.debug("last speech: {}".format(lastSpeech))
-		return lastSpeech
+		return self._getJoinedBaseStringsFromCommands(self._nvdaSpeech[-1])
+
+	def get_all_speech(self):
+		speechCommands = [c for commands in self._nvdaSpeech for c in commands]
+		return self._getJoinedBaseStringsFromCommands(speechCommands)
+
+	def clear_speech_cache(self):
+		self._nvdaSpeech = [
+			[""],  # initialise with an empty string, this allows for access via [-1]. This is equiv to no speech.
+		]
 
 
 class SystemTestSpyServer(object):
diff --git a/tests/system/variables.py b/tests/system/variables.py
new file mode 100644
index 00000000000..c90ecb136cd
--- /dev/null
+++ b/tests/system/variables.py
@@ -0,0 +1,13 @@
+WELCOME_DIALOG_TEXT = ("Welcome to NVDA\n"
+"dialog\n"
+"Welcome to NVDA!\n"
+"Most commands for controlling NVDA require you to hold down the NVDA key while pressing other keys.\n"
+"By default, the numpad Insert and main Insert keys may both be used as the NVDA key.\n"
+"You can also configure NVDA to use the CapsLock as the NVDA key.\n"
+"Press NVDA+n at any time to activate the NVDA menu.\n"
+"From this menu, you can configure NVDA, get help and access other NVDA functions. Options\n"
+"grouping Keyboard layout:\n"
+"combo box\n"
+"desktop\n"
+"collapsed\n"
+"Alt+k")
\ No newline at end of file

From b0477e8e6ac5401b4f4df9892ce56854f86193cc Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 07:31:49 +0000
Subject: [PATCH 42/95] Copy nvda log for each test

---
 tests/system/libraries/nvdaRobotLib.py | 33 +++++++++++++++++++++-----
 tests/system/readme.md                 | 14 +++++++++--
 2 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 2a259c4e713..0373313c88a 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -1,4 +1,6 @@
 import os
+from os.path import join as pathJoin
+from os.path import abspath
 from timeit import default_timer as timer
 from robotremoteserver import test_remote_server, stop_remote_server
 from robot.libraries.BuiltIn import BuiltIn
@@ -12,13 +14,16 @@
 spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
 spyAlias = "nvdaSpy"
 
+nvdaLogFilePath = abspath("source/nvda.log")
+
+systemTestSourceDir = abspath("tests/system")
+nvdaProfileWorkingDir = pathJoin(systemTestSourceDir, "nvdaProfile")
+nvdaSettingsSourceDir = pathJoin(systemTestSourceDir, "nvdaSettingsFiles")
+
 systemTestSpyFileName = "systemTestSpy.py"
-systemTestSourceDir = os.path.abspath("tests/system")
-nvdaProfileWorkingDir = os.path.join(systemTestSourceDir, "nvdaProfile")
-nvdaSettingsSourceDir = os.path.join(systemTestSourceDir, "nvdaSettingsFiles")
-systemTestSpySource = os.path.join(systemTestSourceDir, "libraries", systemTestSpyFileName)
-systemTestSpyInstallDir = os.path.join(nvdaProfileWorkingDir, "globalPlugins")
-systemTestSpyInstalled = os.path.join(systemTestSpyInstallDir, systemTestSpyFileName)
+systemTestSpySource = pathJoin(systemTestSourceDir, "libraries", systemTestSpyFileName)
+systemTestSpyInstallDir = pathJoin(nvdaProfileWorkingDir, "globalPlugins")
+systemTestSpyInstalled = pathJoin(systemTestSpyInstallDir, systemTestSpyFileName)
 
 class nvdaRobotLib(object):
 
@@ -115,6 +120,21 @@ def wait_for_NVDA_startup_to_complete(self):
 			func=lambda: self._runNvdaSpyKeyword("is_NVDA_startup_complete")
 		)
 
+	def save_NVDA_log(self):
+		"""NVDA logs are saved to the ${OUTPUT DIR}/nvdaTestRunLogs/${SUITE NAME}-${TEST NAME}-nvda.log"""
+		outDir = builtIn.get_variable_value("${OUTPUT DIR}", )
+		suiteName = builtIn.get_variable_value("${SUITE NAME}")
+		testName = builtIn.get_variable_value("${TEST NAME}")
+		outputFileName = "{suite}-{test}-nvda.log"\
+			.format(
+				suite=suiteName,
+				test=testName,
+			).replace(" ", "_")
+		opSys.copy_file(
+			nvdaLogFilePath,
+			pathJoin(outDir, "nvdaTestRunLogs", outputFileName)
+		)
+
 	def quit_NVDA(self):
 		stop_remote_server(spyServerURI, log=False)
 		# remove the spy so that if nvda is run manually against this config it does not interfere.
@@ -125,6 +145,7 @@ def quit_NVDA(self):
 			shell=True,
 		)
 		process.wait_for_process(self.nvdaHandle)
+		self.save_NVDA_log()
 
 	def assert_last_speech(self, expectedSpeech):
 		actualLastSpeech = self._runNvdaSpyKeyword("get_last_speech")
diff --git a/tests/system/readme.md b/tests/system/readme.md
index 531c49c4aff..7112bda3653 100644
--- a/tests/system/readme.md
+++ b/tests/system/readme.md
@@ -23,5 +23,15 @@ These tests should be run from the windows command prompt (cmd.exe).
 From the root directory of your NVDA repository, run:
 
 ```
-python -m robot tests/system/ in CMD.
-```
\ No newline at end of file
+python -m robot tests/system/
+```
+
+To run a single test:
+
+```
+python -m robot --test "name of test here" tests/system/
+```
+
+### Getting the results
+
+The process is displayed in the command prompt, for more information consider the `report.html`, `log.html`, and `output.xml` files.
\ No newline at end of file

From 4f9149e23c5738dc5ceecf88b1aaf4bd5634c33a Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 07:43:56 +0000
Subject: [PATCH 43/95] Try to get systemtest artefacts from appveyor

---
 appveyor.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 9c9bcd000ff..421592601b9 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -103,25 +103,27 @@ before_test:
  - py -m pip install robotremoteserver
  - py -m pip install pyautogui
  - py -m pip install nose 
- - mkdir testResults
+ - mkdir output\testResults
 
 test_script:
  - ps: |
      $errorCode=0
-     py -m nose --with-xunit --xunit-file=testResults/unitTests.xml tests/unit
+     py -m nose --with-xunit --xunit-file=output/testResults/unitTests.xml tests/unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\unitTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
-     py -m robot --loglevel DEBUG -d testResults -x systemTests.xml tests/system/initial.robot
+     py -m robot --loglevel DEBUG -d output/testResults -x systemTests.xml tests/system/
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testResults\systemTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\systemTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
 
 artifacts:
  - path: output\*
  - path: output\*\*
+ - path: output\testResults\*
+ - path: output\testResults\*\*
 
 deploy_script:
  - ps: |

From 36e7ee8fa19fd146629dbc6e6daf8258222b416d Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 08:07:01 +0000
Subject: [PATCH 44/95] Try manually pushing artifacts

---
 appveyor.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/appveyor.yml b/appveyor.yml
index 421592601b9..5b6acdc7d1c 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -110,10 +110,12 @@ test_script:
      $errorCode=0
      py -m nose --with-xunit --xunit-file=output/testResults/unitTests.xml tests/unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
+     Push-AppveyorArtifact .\output\testResults\unitTests.xml
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
      py -m robot --loglevel DEBUG -d output/testResults -x systemTests.xml tests/system/
+     Push-AppveyorArtifact .\output\testResults\
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\systemTests.xml))

From e6a6fbf8800de2375f7d33a4ccc07ed78d18c0f4 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 08:39:57 +0000
Subject: [PATCH 45/95] Try recursing over files and pushing manually

---
 appveyor.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 5b6acdc7d1c..94b37b0e0e8 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -104,6 +104,7 @@ before_test:
  - py -m pip install pyautogui
  - py -m pip install nose 
  - mkdir output\testResults
+ - mkdir output\testResults\system
 
 test_script:
  - ps: |
@@ -114,18 +115,19 @@ test_script:
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
-     py -m robot --loglevel DEBUG -d output/testResults -x systemTests.xml tests/system/
-     Push-AppveyorArtifact .\output\testResults\
+     py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml tests/system/
+     Get-ChildItem ".\output/testResults/system\" -Attributes !Directory -depth 2 |
+      Foreach-Object {
+          Push-AppveyorArtifact $_.FullName
+      }
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\systemTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\system\systemTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
 
 artifacts:
  - path: output\*
  - path: output\*\*
- - path: output\testResults\*
- - path: output\testResults\*\*
 
 deploy_script:
  - ps: |

From 4787753ae6fa39e75fb96875aaf70af58b2509b6 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 20 Jun 2018 10:57:12 +0000
Subject: [PATCH 46/95] Fix issue when nvdaProfile folders do not exist

---
 tests/system/libraries/nvdaRobotLib.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 0373313c88a..d5b070d7e05 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -32,6 +32,7 @@ def __init__(self):
 		self.nvdaHandle = None
 
 	def setup_nvda_profile(self, settingsFileName):
+		opSys.create_directory(systemTestSpyInstallDir)
 		opSys.copy_file(systemTestSpySource, systemTestSpyInstallDir)
 		opSys.copy_file(
 			os.path.join(nvdaSettingsSourceDir, settingsFileName),
@@ -78,6 +79,7 @@ def _connectToRemoteServer(self):
 		Instead we wait until the remote server is available before importing the library and continuing.
 		"""
 
+		process.process_should_be_running(self.nvdaHandle)
 		# Importing the 'Remote' library always succeeds, even when a connection can not be made.
 		# If that happens, then some 'Remote' keyword will fail at some later point.
 		# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.

From e9f2189201cba9b45efedb4dd97f34d4315bb6a2 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 21 Jun 2018 10:11:58 +1000
Subject: [PATCH 47/95] system tests: don't show usage stats dialog in NVDA

---
 tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini | 2 ++
 .../system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
index 386aa4d5d2a..00c3dc42a4a 100644
--- a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
@@ -1,3 +1,5 @@
 schemaVersion = 2
 [general]
 	showWelcomeDialogAtStartup = True
+[update]
+	askedAllowUsageStats = True
diff --git a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
index e62f2942cb4..8abd8f479ad 100644
--- a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
@@ -1,3 +1,5 @@
 schemaVersion = 2
 [general]
 	showWelcomeDialogAtStartup = False
+[update]
+	askedAllowUsageStats = True

From 53219530438a16cd0c8b14701935b81d1b6c65d7 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 21 Jun 2018 10:13:26 +1000
Subject: [PATCH 48/95] appveyor: install NVDA on the build system, and run the
 system tests against the installed copy.

---
 appveyor.yml                           | 10 +++++++++-
 tests/system/libraries/nvdaRobotLib.py | 15 +++++++++------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 94b37b0e0e8..959826a2c04 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -115,7 +115,15 @@ test_script:
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
-     py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml tests/system/
+     $nvdaLauncherFile="output\nvda_"
+     if(!$env:release) {
+      $nvdaLauncherFile+="_snapshot"
+     }
+     $nvdaLauncherFile+="$env_version.exe"
+     $nvdaLauncherfile --install-silent --debug-logging --log-file .\output\nvda_install.log 
+     Push-AppveyorArtifact .\output\nvda_install.log
+     if($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) }
+     py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml -v whichNVDA:installed tests/system/
      Get-ChildItem ".\output/testResults/system\" -Attributes !Directory -depth 2 |
       Foreach-Object {
           Push-AppveyorArtifact $_.FullName
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index d5b070d7e05..7fa094e06dc 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -13,11 +13,16 @@
 spyServerPort = 8270  # is `registered by IANA` for remote server usage. Two ASCII values:'RF'
 spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
 spyAlias = "nvdaSpy"
-
-nvdaLogFilePath = abspath("source/nvda.log")
+whichNVDA=builtIn.get_variable_value("${whichNVDA}", "source")
+if whichNVDA=="source":
+	baseNVDACommandline="pythonw source/nvda.pyw"
+elif whichNVDA=="installed":
+	baseNVDACommandline='"%s"'%os.path.join(os.path.expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
+print baseNVDACommandline
 
 systemTestSourceDir = abspath("tests/system")
 nvdaProfileWorkingDir = pathJoin(systemTestSourceDir, "nvdaProfile")
+nvdaLogFilePath = pathJoin(nvdaProfileWorkingDir,'nvda.log')
 nvdaSettingsSourceDir = pathJoin(systemTestSourceDir, "nvdaSettingsFiles")
 
 systemTestSpyFileName = "systemTestSpy.py"
@@ -51,8 +56,7 @@ def _startNVDAProcess(self):
 		Use debug logging, replacing any current instance, using the system test profile directory
 		"""
 		self.nvdaHandle = handle = process.start_process(
-			"pythonw nvda.pyw --debug-logging -r -c \"{nvdaProfileDir}\"".format(nvdaProfileDir=nvdaProfileWorkingDir),
-			cwd='source',
+			"{baseNVDACommandline} --debug-logging -r -c \"{nvdaProfileDir}\" --log-file \"{nvdaLogFilePath}\"".format(baseNVDACommandline=baseNVDACommandline,nvdaProfileDir=nvdaProfileWorkingDir,nvdaLogFilePath=nvdaLogFilePath),
 			shell=True,
 			alias='nvdaAlias'
 		)
@@ -142,8 +146,7 @@ def quit_NVDA(self):
 		# remove the spy so that if nvda is run manually against this config it does not interfere.
 		self.teardown_nvda_profile()
 		process.run_process(
-			"pythonw nvda.pyw -q --disable-addons",
-			cwd='source',
+			"{baseNVDACommandline} -q --disable-addons".format(baseNVDACommandline=baseNVDACommandline),
 			shell=True,
 		)
 		process.wait_for_process(self.nvdaHandle)

From 13b53972b735556fe052261e8aa17a1156ecdf3a Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 21 Jun 2018 11:08:22 +1000
Subject: [PATCH 49/95] appveyor: install nvda with start-process and wait up
 to 3 minutes for it to complete, failing on timeout.

---
 appveyor.yml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 959826a2c04..61ebab70b34 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -120,9 +120,17 @@ test_script:
       $nvdaLauncherFile+="_snapshot"
      }
      $nvdaLauncherFile+="$env_version.exe"
-     $nvdaLauncherfile --install-silent --debug-logging --log-file .\output\nvda_install.log 
-     Push-AppveyorArtifact .\output\nvda_install.log
-     if($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) }
+     $installerLogFilePath=$(resolve-path .\output\nvda_install.log)
+     $installerProcess=start-process -FilePath "$nvdaLauncherFile" -ArgumentList "--install-silent --debug-logging --log-file $installerLogFilePath" -passthru
+     try {
+      $installerProcess | wait-process -Timeout 180
+      $errorCode=$installerProcess.ExitCode
+     } catch {
+      echo NVDA installer process timed out
+      $errorCode=1
+     }
+          Push-AppveyorArtifact $installerLogFilePath
+     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
      py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml -v whichNVDA:installed tests/system/
      Get-ChildItem ".\output/testResults/system\" -Attributes !Directory -depth 2 |
       Foreach-Object {

From 66b0005a4c2c7092f7952815a1ac682dd5ab9b44 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 21 Jun 2018 11:53:15 +1000
Subject: [PATCH 50/95] fix appveyor errors

---
 appveyor.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 61ebab70b34..ca3c060dfc3 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -115,12 +115,14 @@ test_script:
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
-     $nvdaLauncherFile="output\nvda_"
+     $nvdaLauncherFile=".\output\nvda"
      if(!$env:release) {
       $nvdaLauncherFile+="_snapshot"
      }
-     $nvdaLauncherFile+="$env_version.exe"
-     $installerLogFilePath=$(resolve-path .\output\nvda_install.log)
+     $nvdaLauncherFile+="_${env:version}.exe"
+     echo NVDALauncherFile: $NVDALauncherFile
+     $outputDir=$(resolve-path .\output)
+          $installerLogFilePath="$outputDir\nvda_install.log"
      $installerProcess=start-process -FilePath "$nvdaLauncherFile" -ArgumentList "--install-silent --debug-logging --log-file $installerLogFilePath" -passthru
      try {
       $installerProcess | wait-process -Timeout 180

From 6600242a2338a813c0cd731e9db8388e895a35f0 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 21 Jun 2018 12:33:03 +1000
Subject: [PATCH 51/95] Temprarily include robotremoteserver in NVDA for system
 tests. Must figure out a better solution.

---
 appveyor.yml    | 1 +
 source/setup.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index ca3c060dfc3..de9d5238fb3 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -55,6 +55,7 @@ install:
  - type ssh_known_hosts >> %userprofile%\.ssh\known_hosts
  - cd ..
  - git submodule update --init
+ - py -m pip install robotremoteserver
 
 build_script:
  - ps: |
diff --git a/source/setup.py b/source/setup.py
index 92726fdfa0a..761cbc33b25 100755
--- a/source/setup.py
+++ b/source/setup.py
@@ -207,7 +207,7 @@ def getRecursiveDataFiles(dest,source,excludes=()):
 		# #3368: bisect was implicitly included with Python 2.7.3, but isn't with 2.7.5.
 		# Also, the service executable used win32api, which some add-ons use for various purposes.
 		# Explicitly include them so we don't break some add-ons.
-		"includes": ["nvdaBuiltin", "bisect", "win32api"],
+		"includes": ["nvdaBuiltin", "bisect", "win32api","robotremoteserver"],
 	}},
 	data_files=[
 		(".",glob("*.dll")+glob("*.manifest")+["builtin.dic"]),

From d02fe756d5873352486c32e0e0c9bb72f290274b Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Thu, 21 Jun 2018 12:47:01 +1000
Subject: [PATCH 52/95] Breakup up appveyor test code into separate powershell
 code chunks to better show build progress.

---
 appveyor.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/appveyor.yml b/appveyor.yml
index de9d5238fb3..08730f82dcf 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -116,6 +116,8 @@ test_script:
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+ - ps: |
+     $errorCode=0
      $nvdaLauncherFile=".\output\nvda"
      if(!$env:release) {
       $nvdaLauncherFile+="_snapshot"
@@ -134,6 +136,7 @@ test_script:
      }
           Push-AppveyorArtifact $installerLogFilePath
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+ - ps: |
      py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml -v whichNVDA:installed tests/system/
      Get-ChildItem ".\output/testResults/system\" -Attributes !Directory -depth 2 |
       Foreach-Object {

From c985677445529a91519497952bbd283b3853326f Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 13:26:32 +1000
Subject: [PATCH 53/95] system tests: use silence synth driver.

---
 tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini | 2 ++
 .../system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
index 00c3dc42a4a..9e065469ab4 100644
--- a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
@@ -3,3 +3,5 @@ schemaVersion = 2
 	showWelcomeDialogAtStartup = True
 [update]
 	askedAllowUsageStats = True
+[speech]
+	synth=silence
diff --git a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
index 8abd8f479ad..21132e92528 100644
--- a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
@@ -3,3 +3,5 @@ schemaVersion = 2
 	showWelcomeDialogAtStartup = False
 [update]
 	askedAllowUsageStats = True
+[speech]
+	synth=silence

From 5c4d2846eac7abc6c8397bbe7cfd664c67021e53 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 15:42:55 +1000
Subject: [PATCH 54/95] system test for exit dialog: just sleep instead of
 varifying dialog.

---
 tests/system/initial.robot | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 14c76487893..b16f5263278 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -17,7 +17,7 @@ Ensure NVDA runs at all
 
 Ensure NVDA quits from keyboard
 	send key	insert	q
-	wait for foreground	Exit NVDA
+	sleep 1
 	send key	enter
 	wait for process	nvdaAlias	timeout=5 sec
 

From 844c73b7ef8bc2f152474306f5eebbeb52c956c9 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 15:57:07 +1000
Subject: [PATCH 55/95] system tests: don't wait for exit dialog, rather just
 confirm that the NVDA process definitlly exited.

---
 tests/system/initial.robot | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index b16f5263278..309df98924a 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -17,10 +17,11 @@ Ensure NVDA runs at all
 
 Ensure NVDA quits from keyboard
 	send key	insert	q
-	sleep 1
 	send key	enter
-	wait for process	nvdaAlias	timeout=5 sec
+	${result} =	wait for process	nvdaAlias	timeout=5 sec
+	process should be stopped	nvdaAlias
 
 Can read the welcome dialog
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
-	assert all speech	${WELCOME_DIALOG_TEXT}
\ No newline at end of file
+	assert all speech	${WELCOME_DIALOG_TEXT}
+	send key	enter

From 78432cf42b77effeee2a4517c94e6e5a62a50aba Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 16:19:54 +1000
Subject: [PATCH 56/95] appveyor: move NVDA installation to before_test
 section.

---
 appveyor.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index e2811fd90a1..12f08115334 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -118,16 +118,6 @@ before_test:
  - py -m pip install nose 
  - mkdir output\testResults
  - mkdir output\testResults\system
-
-test_script:
- - ps: |
-     $errorCode=0
-     py -m nose --with-xunit --xunit-file=output/testResults/unitTests.xml tests/unit
-     if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
-     Push-AppveyorArtifact .\output\testResults\unitTests.xml
-     $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
-     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
  - ps: |
      $errorCode=0
      $nvdaLauncherFile=".\output\nvda"
@@ -148,6 +138,16 @@ test_script:
      }
           Push-AppveyorArtifact $installerLogFilePath
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+
+test_script:
+ - ps: |
+     $errorCode=0
+     py -m nose --with-xunit --xunit-file=output/testResults/unitTests.xml tests/unit
+     if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
+     Push-AppveyorArtifact .\output\testResults\unitTests.xml
+     $wc = New-Object 'System.Net.WebClient'
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
+     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
  - ps: |
      py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml -v whichNVDA:installed tests/system/
      Get-ChildItem ".\output/testResults/system\" -Attributes !Directory -depth 2 |

From 221208e0ce25ea9801eb6a35b3a605b01ec5063a Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 16:37:36 +1000
Subject: [PATCH 57/95] ensure quit system test: exit from the NVDA menu.

---
 tests/system/initial.robot | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 309df98924a..623e4d2138b 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -16,9 +16,11 @@ Ensure NVDA runs at all
 	process should be running	nvdaAlias
 
 Ensure NVDA quits from keyboard
-	send key	insert	q
+	send key	insert	n
+	send key	x
+	sleep	1
 	send key	enter
-	${result} =	wait for process	nvdaAlias	timeout=5 sec
+	wait for process	nvdaAlias	timeout=5 sec
 	process should be stopped	nvdaAlias
 
 Can read the welcome dialog

From 7bd259ad2529ec178d834019719bddf52986f9dd Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 16:51:44 +1000
Subject: [PATCH 58/95] system tests: try alt tabbing to the exit dialog???

---
 tests/system/initial.robot | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 623e4d2138b..dce868184a7 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -19,6 +19,7 @@ Ensure NVDA quits from keyboard
 	send key	insert	n
 	send key	x
 	sleep	1
+	send key	alt	shift	tab
 	send key	enter
 	wait for process	nvdaAlias	timeout=5 sec
 	process should be stopped	nvdaAlias

From 6a757e1ccba2c1b3745159b926144ebda2abe001 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 17:15:29 +1000
Subject: [PATCH 59/95] More debugging for system tests.

---
 source/IAccessibleHandler.py |  4 ++++
 tests/system/initial.robot   | 10 +++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py
index 89a565bec40..8b78e9c965d 100644
--- a/source/IAccessibleHandler.py
+++ b/source/IAccessibleHandler.py
@@ -539,6 +539,10 @@ def winEventToNVDAEvent(eventID,window,objectID,childID,useCache=True):
 
 def winEventCallback(handle,eventID,window,objectID,childID,threadID,timestamp):
 	try:
+		if eventID==winUser.EVENT_OBJECT_FOCUS:
+			log.info("focus %s, %s"%(winUser.getClassName(window),winUser.getWindowText(window)))
+		if eventID==winUser.EVENT_SYSTEM_FOREGROUND:
+			log.info("foreground %s, %s"%(winUser.getClassName(window),winUser.getWindowText(window)))
 		#Ignore all object IDs from alert onwards (sound, nativeom etc) as we don't support them
 		if objectID<=winUser.OBJID_ALERT: 
 			return
diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index dce868184a7..9f81c393305 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -16,11 +16,15 @@ Ensure NVDA runs at all
 	process should be running	nvdaAlias
 
 Ensure NVDA quits from keyboard
-	send key	insert	n
-	send key	x
+	send key	insert	tab
+	send key	insert	q
 	sleep	1
-	send key	alt	shift	tab
+	send key	insert	tab
+	wait for foreground	Exit NVDA
+	send key	insert	tab
 	send key	enter
+	sleep	1
+	send key	insert	tab
 	wait for process	nvdaAlias	timeout=5 sec
 	process should be stopped	nvdaAlias
 

From 6edf1e03b386bdad10772e117375d85a1d3b7055 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 17:38:52 +1000
Subject: [PATCH 60/95] IAccessibleHandler: more debugging.

---
 source/IAccessibleHandler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py
index 8b78e9c965d..811ba8b5f9f 100644
--- a/source/IAccessibleHandler.py
+++ b/source/IAccessibleHandler.py
@@ -681,11 +681,14 @@ def processFocusNVDAEvent(obj,force=False):
 	@return: C{True} if the focus event is valid and was queued, C{False} otherwise.
 	@rtype: boolean
 	"""
+	log.info("IAccessible NVDAObject focus:\n%s"%"\n".join(obj.devInfo))
 	if not force and isinstance(obj,NVDAObjects.IAccessible.IAccessible):
 		focus=eventHandler.lastQueuedFocusObject
 		if isinstance(focus,NVDAObjects.IAccessible.IAccessible) and focus.isDuplicateIAccessibleEvent(obj):
+			log.info("duplicate focus event")
 			return True
 		if not obj.shouldAllowIAccessibleFocusEvent:
+			log.info("focus event not allowed")
 			return False
 	eventHandler.queueEvent('gainFocus',obj)
 	return True

From 167b362a0ca33c4d318add2b344ad0e2708ea242 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 19:09:15 +1000
Subject: [PATCH 61/95] IAccessibleHandler: more debugging.

---
 source/IAccessibleHandler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py
index 811ba8b5f9f..85af6966791 100644
--- a/source/IAccessibleHandler.py
+++ b/source/IAccessibleHandler.py
@@ -876,6 +876,8 @@ def pumpAll():
 			if not focus.shouldAcceptShowHideCaretEvent:
 				continue
 		elif not eventHandler.shouldAcceptEvent(winEventIDsToNVDAEventNames[winEvent[0]], windowHandle=winEvent[1]):
+			if winEvent[0]==winUser.EVENT_OBJECT_FOCUS:
+				log.info("Dropped focus winevent")
 			continue
 		#We want to only pass on one focus event to NVDA, but we always want to use the most recent possible one 
 		if winEvent[0] in (winUser.EVENT_OBJECT_FOCUS,winUser.EVENT_SYSTEM_FOREGROUND):

From 27446cfd0f17652ebdb7b03b107cebf3b8baade4 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 19:19:19 +1000
Subject: [PATCH 62/95] eventHandler: more debugging.

---
 source/eventHandler.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/eventHandler.py b/source/eventHandler.py
index af95d062704..21ad233f5a0 100755
--- a/source/eventHandler.py
+++ b/source/eventHandler.py
@@ -273,6 +273,8 @@ def shouldAcceptEvent(eventName, windowHandle=None):
 			return True
 
 	fg = winUser.getForegroundWindow()
+	if eventName=="gainFocus":
+		log.info("Checking focus event for %s against foreground of %s, %s"%(wClass,fg,winUser.getClassName(fg)))
 	if wClass == "NetUIHWND" and winUser.getClassName(fg) == "Net UI Tool Window Layered":
 		# #5504: In Office >= 2013 with the ribbon showing only tabs,
 		# when a tab is expanded, the window we get from the focus object is incorrect.
@@ -291,4 +293,6 @@ def shouldAcceptEvent(eventName, windowHandle=None):
 		# This window or its root is a topmost window.
 		# This includes menus, combo box pop-ups and the task switching list.
 		return True
+	if eventName=="gainfocus::
+		log.info("Filtered out focus event")
 	return False

From ce6474a89104f03c5c5d810425d6c02ae882bab4 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 19:31:57 +1000
Subject: [PATCH 63/95] Up max foreground defer count to 10.

---
 source/IAccessibleHandler.py | 2 +-
 source/eventHandler.py       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py
index 85af6966791..4bb8a867731 100644
--- a/source/IAccessibleHandler.py
+++ b/source/IAccessibleHandler.py
@@ -137,7 +137,7 @@ def flushEvents(self):
 
 # #3831: Stuff related to deferring of events for foreground changes.
 # See pumpAll for details.
-MAX_FOREGROUND_DEFERS=2
+MAX_FOREGROUND_DEFERS=10
 _deferUntilForegroundWindow = None
 _foregroundDefers = 0
 
diff --git a/source/eventHandler.py b/source/eventHandler.py
index 21ad233f5a0..a260c9183b2 100755
--- a/source/eventHandler.py
+++ b/source/eventHandler.py
@@ -293,6 +293,6 @@ def shouldAcceptEvent(eventName, windowHandle=None):
 		# This window or its root is a topmost window.
 		# This includes menus, combo box pop-ups and the task switching list.
 		return True
-	if eventName=="gainfocus::
+	if eventName=="gainfocus":
 		log.info("Filtered out focus event")
 	return False

From 86aeb0697d30b18837b1848d9308a7031e1eebdd Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 19:57:33 +1000
Subject: [PATCH 64/95] Up foreground defers to 100 and log them.

---
 source/IAccessibleHandler.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py
index 4bb8a867731..0af8254d669 100644
--- a/source/IAccessibleHandler.py
+++ b/source/IAccessibleHandler.py
@@ -137,7 +137,7 @@ def flushEvents(self):
 
 # #3831: Stuff related to deferring of events for foreground changes.
 # See pumpAll for details.
-MAX_FOREGROUND_DEFERS=10
+MAX_FOREGROUND_DEFERS=100
 _deferUntilForegroundWindow = None
 _foregroundDefers = 0
 
@@ -850,10 +850,12 @@ def pumpAll():
 		# but GetForegroundWindow() takes a short while to return this new foreground.
 		if _foregroundDefers<MAX_FOREGROUND_DEFERS and winUser.getForegroundWindow()!=_deferUntilForegroundWindow:
 			# Wait a core cycle before handling events to give the foreground window time to update.
+			log.info("Defer foreground")
 			core.requestPump()
 			_foregroundDefers+=1
 			return
 		else:
+			log.info("allowing foreground")
 			# Either the foreground window is now correct
 			# or we've already had the maximum number of defers.
 			# (Sometimes, foreground events are fired even when the foreground hasn't actually changed.)

From 8bfbf746f408cefa69da26a3f51826210178efa0 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 26 Jun 2018 20:38:50 +1000
Subject: [PATCH 65/95] IAccessibleHandler: drop foreground defers back down
 and  allow welcome dialog in  quit from keyboard system test.

---
 source/IAccessibleHandler.py | 2 +-
 tests/system/initial.robot   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py
index 0af8254d669..07e67a54e5f 100644
--- a/source/IAccessibleHandler.py
+++ b/source/IAccessibleHandler.py
@@ -137,7 +137,7 @@ def flushEvents(self):
 
 # #3831: Stuff related to deferring of events for foreground changes.
 # See pumpAll for details.
-MAX_FOREGROUND_DEFERS=100
+MAX_FOREGROUND_DEFERS=2
 _deferUntilForegroundWindow = None
 _foregroundDefers = 0
 
diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 9f81c393305..ef240458adc 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -16,6 +16,7 @@ Ensure NVDA runs at all
 	process should be running	nvdaAlias
 
 Ensure NVDA quits from keyboard
+	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
 	send key	insert	tab
 	send key	insert	q
 	sleep	1

From 80276820c6a5acea8b688a826cf114ded5ab2722 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Wed, 27 Jun 2018 02:49:30 +1000
Subject: [PATCH 66/95] system tests for quit with keyboard: watch for and
 close welcome dialog first.

---
 tests/system/initial.robot | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index ef240458adc..7eff749f2ca 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -17,15 +17,11 @@ Ensure NVDA runs at all
 
 Ensure NVDA quits from keyboard
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
-	send key	insert	tab
+	wait for foreground	Welcome to NVDA
+	send key	enter
 	send key	insert	q
-	sleep	1
-	send key	insert	tab
 	wait for foreground	Exit NVDA
-	send key	insert	tab
 	send key	enter
-	sleep	1
-	send key	insert	tab
 	wait for process	nvdaAlias	timeout=5 sec
 	process should be stopped	nvdaAlias
 

From 2913125bec7106cdc56885f9b18048af94518699 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Wed, 27 Jun 2018 04:10:20 +1000
Subject: [PATCH 67/95] quit with keyboard system test: wait longer for NVDA to
 exit.

---
 tests/system/initial.robot | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 7eff749f2ca..34c5f104f9e 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -22,7 +22,7 @@ Ensure NVDA quits from keyboard
 	send key	insert	q
 	wait for foreground	Exit NVDA
 	send key	enter
-	wait for process	nvdaAlias	timeout=5 sec
+	wait for process	nvdaAlias	timeout=10 sec
 	process should be stopped	nvdaAlias
 
 Can read the welcome dialog

From e6e81882be1bcc19c184a69abcbf664950153d3c Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Wed, 27 Jun 2018 11:00:02 +1000
Subject: [PATCH 68/95] Restore IAccessibleHandler and eventHandler.

---
 source/IAccessibleHandler.py | 11 -----------
 source/eventHandler.py       |  4 ----
 2 files changed, 15 deletions(-)

diff --git a/source/IAccessibleHandler.py b/source/IAccessibleHandler.py
index 07e67a54e5f..89a565bec40 100644
--- a/source/IAccessibleHandler.py
+++ b/source/IAccessibleHandler.py
@@ -539,10 +539,6 @@ def winEventToNVDAEvent(eventID,window,objectID,childID,useCache=True):
 
 def winEventCallback(handle,eventID,window,objectID,childID,threadID,timestamp):
 	try:
-		if eventID==winUser.EVENT_OBJECT_FOCUS:
-			log.info("focus %s, %s"%(winUser.getClassName(window),winUser.getWindowText(window)))
-		if eventID==winUser.EVENT_SYSTEM_FOREGROUND:
-			log.info("foreground %s, %s"%(winUser.getClassName(window),winUser.getWindowText(window)))
 		#Ignore all object IDs from alert onwards (sound, nativeom etc) as we don't support them
 		if objectID<=winUser.OBJID_ALERT: 
 			return
@@ -681,14 +677,11 @@ def processFocusNVDAEvent(obj,force=False):
 	@return: C{True} if the focus event is valid and was queued, C{False} otherwise.
 	@rtype: boolean
 	"""
-	log.info("IAccessible NVDAObject focus:\n%s"%"\n".join(obj.devInfo))
 	if not force and isinstance(obj,NVDAObjects.IAccessible.IAccessible):
 		focus=eventHandler.lastQueuedFocusObject
 		if isinstance(focus,NVDAObjects.IAccessible.IAccessible) and focus.isDuplicateIAccessibleEvent(obj):
-			log.info("duplicate focus event")
 			return True
 		if not obj.shouldAllowIAccessibleFocusEvent:
-			log.info("focus event not allowed")
 			return False
 	eventHandler.queueEvent('gainFocus',obj)
 	return True
@@ -850,12 +843,10 @@ def pumpAll():
 		# but GetForegroundWindow() takes a short while to return this new foreground.
 		if _foregroundDefers<MAX_FOREGROUND_DEFERS and winUser.getForegroundWindow()!=_deferUntilForegroundWindow:
 			# Wait a core cycle before handling events to give the foreground window time to update.
-			log.info("Defer foreground")
 			core.requestPump()
 			_foregroundDefers+=1
 			return
 		else:
-			log.info("allowing foreground")
 			# Either the foreground window is now correct
 			# or we've already had the maximum number of defers.
 			# (Sometimes, foreground events are fired even when the foreground hasn't actually changed.)
@@ -878,8 +869,6 @@ def pumpAll():
 			if not focus.shouldAcceptShowHideCaretEvent:
 				continue
 		elif not eventHandler.shouldAcceptEvent(winEventIDsToNVDAEventNames[winEvent[0]], windowHandle=winEvent[1]):
-			if winEvent[0]==winUser.EVENT_OBJECT_FOCUS:
-				log.info("Dropped focus winevent")
 			continue
 		#We want to only pass on one focus event to NVDA, but we always want to use the most recent possible one 
 		if winEvent[0] in (winUser.EVENT_OBJECT_FOCUS,winUser.EVENT_SYSTEM_FOREGROUND):
diff --git a/source/eventHandler.py b/source/eventHandler.py
index a260c9183b2..af95d062704 100755
--- a/source/eventHandler.py
+++ b/source/eventHandler.py
@@ -273,8 +273,6 @@ def shouldAcceptEvent(eventName, windowHandle=None):
 			return True
 
 	fg = winUser.getForegroundWindow()
-	if eventName=="gainFocus":
-		log.info("Checking focus event for %s against foreground of %s, %s"%(wClass,fg,winUser.getClassName(fg)))
 	if wClass == "NetUIHWND" and winUser.getClassName(fg) == "Net UI Tool Window Layered":
 		# #5504: In Office >= 2013 with the ribbon showing only tabs,
 		# when a tab is expanded, the window we get from the focus object is incorrect.
@@ -293,6 +291,4 @@ def shouldAcceptEvent(eventName, windowHandle=None):
 		# This window or its root is a topmost window.
 		# This includes menus, combo box pop-ups and the task switching list.
 		return True
-	if eventName=="gainfocus":
-		log.info("Filtered out focus event")
 	return False

From 9fd13f4361231422b70f712c0b92c2e60956c8c5 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 27 Jun 2018 01:08:41 +0000
Subject: [PATCH 69/95] Not able to get an index returned from
 get_index_of_speech

---
 tests/system/initial.robot                    | 15 ++-
 tests/system/libraries/nvdaRobotLib.py        | 98 ++++++++++++++++---
 tests/system/libraries/systemTestSpy.py       | 50 +++++++---
 .../standard-doShowWelcomeDialog.ini          |  2 +
 .../standard-dontShowWelcomeDialog.ini        |  2 +
 tests/system/variables.py                     | 36 ++++---
 6 files changed, 161 insertions(+), 42 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 14c76487893..aca82845a9c 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -15,12 +15,23 @@ Variables	variables.py
 Ensure NVDA runs at all
 	process should be running	nvdaAlias
 
-Ensure NVDA quits from keyboard
+Ensure NVDA quits from keyboard2
+	reset get all speech
 	send key	insert	q
-	wait for foreground	Exit NVDA
+	wait for speech to start and finish
+	assert speech after action	${QUIT_DIALOG_TEXT}
+	send key	enter
+	wait for process	nvdaAlias	timeout=5 sec
+
+Ensure NVDA quits from keyboard
+	${exit_nvda_dialog} = 	catenate	SEPARATOR=\n	Exit NVDA	dialog
+	${INDEX} =	wait_for_specific_speech_after_action	${exit_nvda_dialog}	send key	insert	q
+	wait for speech to finish
+	assert speech since index	${INDEX}	${QUIT_DIALOG_TEXT}
 	send key	enter
 	wait for process	nvdaAlias	timeout=5 sec
 
 Can read the welcome dialog
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
+	wait for speech to finish
 	assert all speech	${WELCOME_DIALOG_TEXT}
\ No newline at end of file
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index d5b070d7e05..bbb875e1c3c 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -58,16 +58,24 @@ def _startNVDAProcess(self):
 		)
 		return handle
 
-	def _blockUntilReturnsTrue(self, giveUpAfterSeconds, intervalBetweenSeconds, errorMessage, func):
+	def _blockUntilReturnsTrue(self, func, giveUpAfterSeconds, intervalBetweenSeconds=0.1, errorMessage=None):
+		"""Call 'func' every 'intervalBetweenSeconds' seconds until 'func' returns True until
+		'giveUpAfterSeconds' is reached.
+		If 'func' does not return true and 'errorMessage' is provided, a RuntimeError is raised
+		using the provided errorMessage.
+		@return True if 'func' returns True before 'giveUpAfterSeconds' is reached, else False
+		"""
 		startTime = timer()
 		lastRunTime = startTime - intervalBetweenSeconds+1  # ensure we start trying immediately
 		while (timer() - startTime) < giveUpAfterSeconds:
 			if (timer() - lastRunTime) > intervalBetweenSeconds:
 				lastRunTime = timer()
 				if func():
-					break
+					return True
 		else:
-			raise RuntimeError(errorMessage)
+			if errorMessage:
+				raise RuntimeError(errorMessage)
+			return False
 
 	def _connectToRemoteServer(self):
 		"""Connects to the nvdaSpyServer
@@ -78,16 +86,13 @@ def _connectToRemoteServer(self):
 				No connection could be made because the target machine actively refused it"
 		Instead we wait until the remote server is available before importing the library and continuing.
 		"""
-
-		process.process_should_be_running(self.nvdaHandle)
 		# Importing the 'Remote' library always succeeds, even when a connection can not be made.
 		# If that happens, then some 'Remote' keyword will fail at some later point.
 		# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.
 		self._blockUntilReturnsTrue(
+			func=lambda: test_remote_server(spyServerURI, log=False),
 			giveUpAfterSeconds=10,
-			intervalBetweenSeconds=0.1,
 			errorMessage="Unable to connect to nvdaSpy",
-			func=lambda: test_remote_server(spyServerURI, log=False)
 		)
 
 		builtIn.import_library(
@@ -116,10 +121,9 @@ def start_NVDA(self, settingsFileName):
 
 	def wait_for_NVDA_startup_to_complete(self):
 		self._blockUntilReturnsTrue(
+			func=lambda: self._runNvdaSpyKeyword("is_NVDA_startup_complete"),
 			giveUpAfterSeconds=10,
-			intervalBetweenSeconds=0.1,
 			errorMessage="Unable to connect to nvdaSpy",
-			func=lambda: self._runNvdaSpyKeyword("is_NVDA_startup_complete")
 		)
 
 	def save_NVDA_log(self):
@@ -151,12 +155,74 @@ def quit_NVDA(self):
 
 	def assert_last_speech(self, expectedSpeech):
 		actualLastSpeech = self._runNvdaSpyKeyword("get_last_speech")
-		builtIn.should_be_equal_as_strings(actualLastSpeech, expectedSpeech)
+		builtIn.should_be_equal_as_strings(
+			actualLastSpeech,
+			expectedSpeech,
+			msg="Actual speech != Expected speech"
+		)
 
 	def assert_all_speech(self, expectedSpeech):
-			actualSpeech = self._runNvdaSpyKeyword("get_all_speech")
-			builtIn.should_be_equal_as_strings(
-				actualSpeech,
-				expectedSpeech,
-				msg="Actual speech != to expected speech.",
-			)
+		actualSpeech = self._runNvdaSpyKeyword("get_all_speech")
+		builtIn.should_be_equal_as_strings(
+			actualSpeech,
+			expectedSpeech,
+			msg="Actual speech != Expected speech",
+		)
+
+	def assert_speech_since_index(self, index, expectedSpeech):
+		actualSpeech = self._runNvdaSpyKeyword("get_speech_since_index", index)
+		builtIn.should_be_equal_as_strings(
+			actualSpeech,
+			expectedSpeech,
+			msg="Actual speech != Expected speech",
+		)
+
+	def wait_for_next_speech(self, maxWaitSeconds=5, raiseErrorOnTimeout=True):
+		return self._blockUntilReturnsTrue(
+			func=lambda: self._runNvdaSpyKeyword("has_speech_occurred_since_last_check"),
+			giveUpAfterSeconds=maxWaitSeconds,
+			errorMessage="Speech did not start before timeout" if raiseErrorOnTimeout else None
+		)
+
+	def has_speech_finished(self):
+		speechOccurred = self.wait_for_next_speech(maxWaitSeconds=2, raiseErrorOnTimeout=False)
+		return not speechOccurred
+
+	def wait_for_speech_to_finish(self, maxWaitSeconds=5):
+		self._blockUntilReturnsTrue(
+			func=self.has_speech_finished,
+			giveUpAfterSeconds=maxWaitSeconds,
+			errorMessage="Speech did not finish before timeout"
+		)
+
+	def wait_for_speech_to_start_and_finish(self):
+		self.wait_for_next_speech()
+		self.wait_for_speech_to_finish()
+
+	def reset_get_all_speech(self):
+		self._runNvdaSpyKeyword("reset_all_speech_index")
+
+	def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
+		sinceIndex = 0 if not sinceIndex else sinceIndex
+		def trueIfSpeechIndex():
+			index = self._runNvdaSpyKeyword("get_index_of_speech", speech, sinceIndex)
+			builtIn.log_to_console("value of index: ".format(index))
+			return index > 0
+
+		self._blockUntilReturnsTrue(
+			func=trueIfSpeechIndex,
+			giveUpAfterSeconds=maxWaitSeconds,
+			errorMessage="Specific speech did not occur before timeout: {}".format(speech)
+		)
+
+	def wait_for_specific_speech_after_action(self, speech, action, *args):
+		self.wait_for_speech_to_finish()
+		index = self._runNvdaSpyKeyword("get_speech_index")
+		builtIn.run_keyword(action, *args)
+		self.wait_for_specific_speech(speech, index)
+		return self._runNvdaSpyKeyword("get_index_of_speech", speech, index)
+
+	def wait_for_any_speech_after_action(self, action, *args):
+		self.wait_for_speech_to_finish()
+		builtIn.run_keyword(action, *args)
+		self.wait_for_next_speech()
\ No newline at end of file
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index 942fa571cde..9e90773111c 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -8,37 +8,65 @@ class SystemTestSpy:
 	def __init__(self):
 		self._nvdaStartupComplete = False
 		from core import postNvdaStartup
-		from speech import preSpeech
 		postNvdaStartup.register(self._onNvdaStartupComplete)
+		from speech import preSpeech
 		preSpeech.register(self._onNvdaSpeech)
-		self.clear_speech_cache()  # set self._nvdaSpeech
+		self._nvdaSpeech = [
+			[""],  # initialise with an empty string, this allows for access via [-1]. This is equiv to no speech.
+		]
+		self._allSpeechStartIndex = 0
+		self._speechOccurred = False
 
 	def _onNvdaStartupComplete(self):
 		self._nvdaStartupComplete = True
-		self.clear_speech_cache()
+		self.reset_all_speech_index()
 
 	def _onNvdaSpeech(self, speechSequence=None):
 		if not speechSequence: return
+		self._speechOccurred = True
 		self._nvdaSpeech.append(speechSequence)
 
-	def is_NVDA_startup_complete(self):
-		return self._nvdaStartupComplete
-
 	def _getJoinedBaseStringsFromCommands(self, speechCommandArray):
 		baseStrings = [c.strip() for c in speechCommandArray if isinstance(c, basestring)]
 		return ' '.join(baseStrings).replace("  ", "\n").strip()
 
+	# Start of Robot library API
+
+	def is_NVDA_startup_complete(self):
+		return self._nvdaStartupComplete
+
+	def has_speech_occurred_since_last_check(self):
+		speechOccurred = self._speechOccurred
+		self._speechOccurred = False
+		return speechOccurred
+
 	def get_last_speech(self):
 		return self._getJoinedBaseStringsFromCommands(self._nvdaSpeech[-1])
 
 	def get_all_speech(self):
-		speechCommands = [c for commands in self._nvdaSpeech for c in commands]
+		return self.get_speech_since_index(self._allSpeechStartIndex)
+
+	def get_speech_since_index(self, speechIndex):
+		speechCommands = [c for commands in self._nvdaSpeech[speechIndex:] for c in commands]
 		return self._getJoinedBaseStringsFromCommands(speechCommands)
 
-	def clear_speech_cache(self):
-		self._nvdaSpeech = [
-			[""],  # initialise with an empty string, this allows for access via [-1]. This is equiv to no speech.
-		]
+	def get_speech_index(self):
+		return len(self._nvdaSpeech) -1
+
+	def reset_all_speech_index(self):
+		self._allSpeechStartIndex = self.get_speech_index()
+
+	def get_index_of_speech(self, speech, indexHint=0):
+		log.debug("indexHint is: {}, speech is: {}".format(indexHint, speech))
+		for index, commands in enumerate(self._nvdaSpeech[indexHint:]):
+			index = index+indexHint
+			baseStrings = [c.strip().replace("  ", "\n") for c in commands if isinstance(c, basestring)]
+			log.debug("baseStrings: \n{}".format(repr(baseStrings)))
+			if speech in baseStrings:
+				log.debug("at index: {}, Found: {}".format(index, speech))
+				return index
+		return 0
+
 
 
 class SystemTestSpyServer(object):
diff --git a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
index 386aa4d5d2a..2a7a5e0a840 100644
--- a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
@@ -1,3 +1,5 @@
 schemaVersion = 2
 [general]
 	showWelcomeDialogAtStartup = True
+[speech]
+	synth = silence
diff --git a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
index e62f2942cb4..b18e288220b 100644
--- a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
@@ -1,3 +1,5 @@
 schemaVersion = 2
 [general]
 	showWelcomeDialogAtStartup = False
+[speech]
+	synth = silence
diff --git a/tests/system/variables.py b/tests/system/variables.py
index c90ecb136cd..90196ef1eb7 100644
--- a/tests/system/variables.py
+++ b/tests/system/variables.py
@@ -1,13 +1,23 @@
-WELCOME_DIALOG_TEXT = ("Welcome to NVDA\n"
-"dialog\n"
-"Welcome to NVDA!\n"
-"Most commands for controlling NVDA require you to hold down the NVDA key while pressing other keys.\n"
-"By default, the numpad Insert and main Insert keys may both be used as the NVDA key.\n"
-"You can also configure NVDA to use the CapsLock as the NVDA key.\n"
-"Press NVDA+n at any time to activate the NVDA menu.\n"
-"From this menu, you can configure NVDA, get help and access other NVDA functions. Options\n"
-"grouping Keyboard layout:\n"
-"combo box\n"
-"desktop\n"
-"collapsed\n"
-"Alt+k")
\ No newline at end of file
+WELCOME_DIALOG_TEXT = (
+	"Welcome to NVDA\n"
+	"dialog\n"
+	"Welcome to NVDA!\n"
+	"Most commands for controlling NVDA require you to hold down the NVDA key while pressing other keys.\n"
+	"By default, the numpad Insert and main Insert keys may both be used as the NVDA key.\n"
+	"You can also configure NVDA to use the CapsLock as the NVDA key.\n"
+	"Press NVDA+n at any time to activate the NVDA menu.\n"
+	"From this menu, you can configure NVDA, get help and access other NVDA functions. Options\n"
+	"grouping Keyboard layout:\n"
+	"combo box\n"
+	"desktop\n"
+	"collapsed\n"
+	"Alt+k"
+)
+QUIT_DIALOG_TEXT = (
+	"Exit NVDA\n"
+	"dialog What would you like to do?\n"
+	"combo box\n"
+	"Exit\n"
+	"collapsed\n"
+	"Alt+d\n"
+)
\ No newline at end of file

From 52853fcfd80f47c41d541914672bcc8cfcdb4989 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 27 Jun 2018 06:20:40 +0000
Subject: [PATCH 70/95] tests passing

---
 tests/system/initial.robot              | 12 +---
 tests/system/libraries/nvdaRobotLib.py  | 85 +++++++++++++++++--------
 tests/system/libraries/systemTestSpy.py | 27 +++++---
 tests/system/variables.py               | 21 ++----
 4 files changed, 86 insertions(+), 59 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index aca82845a9c..51da756e82f 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -15,17 +15,9 @@ Variables	variables.py
 Ensure NVDA runs at all
 	process should be running	nvdaAlias
 
-Ensure NVDA quits from keyboard2
-	reset get all speech
-	send key	insert	q
-	wait for speech to start and finish
-	assert speech after action	${QUIT_DIALOG_TEXT}
-	send key	enter
-	wait for process	nvdaAlias	timeout=5 sec
-
 Ensure NVDA quits from keyboard
-	${exit_nvda_dialog} = 	catenate	SEPARATOR=\n	Exit NVDA	dialog
-	${INDEX} =	wait_for_specific_speech_after_action	${exit_nvda_dialog}	send key	insert	q
+	${Exit NVDA dialog} =	catenate	SEPARATOR=${SPACE * 2}	Exit NVDA	dialog
+	${INDEX} =	wait_for_specific_speech_after_action	${Exit NVDA dialog}	send key	insert	q
 	wait for speech to finish
 	assert speech since index	${INDEX}	${QUIT_DIALOG_TEXT}
 	send key	enter
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index bbb875e1c3c..0ffbd8c8b32 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -25,6 +25,24 @@
 systemTestSpyInstallDir = pathJoin(nvdaProfileWorkingDir, "globalPlugins")
 systemTestSpyInstalled = pathJoin(systemTestSpyInstallDir, systemTestSpyFileName)
 
+def _should_be_equal_as_strings(actual, expected, ignore_case=False):
+	try:
+		builtIn.should_be_equal_as_strings(
+			actual,
+			expected,
+			msg="Actual speech != Expected speech",
+			ignore_case=ignore_case
+		)
+	except AssertionError:
+		builtIn.log(
+			"repr of actual vs expected (ignore_case={}):\n{}\nvs\n{}".format(
+				ignore_case,
+				repr(actual),
+				repr(expected)
+			)
+		)
+		raise
+
 class nvdaRobotLib(object):
 
 	def __init__(self):
@@ -77,6 +95,33 @@ def _blockUntilReturnsTrue(self, func, giveUpAfterSeconds, intervalBetweenSecond
 				raise RuntimeError(errorMessage)
 			return False
 
+
+	def _blockUntilConditionMet(
+			self,
+			getValue,
+			giveUpAfterSeconds,
+			shouldStopEvaluator=lambda value: bool(value),
+			intervalBetweenSeconds=0.1,
+			errorMessage=None):
+		"""Repeatedly tries to get a value up until a time limit expires. Tries are separated by
+		a time interval. The call will block until shouldStopEvaluator returns True when given the value,
+		the default evaluator just returns the value converted to a boolean.
+		@return A tuple, (True, value) if evaluator condition is met, otherwise (False, None)
+		@raises RuntimeError if the time limit expires and an errorMessage is given.
+		"""
+		startTime = timer()
+		lastRunTime = startTime - intervalBetweenSeconds+1  # ensure we start trying immediately
+		while (timer() - startTime) < giveUpAfterSeconds:
+			if (timer() - lastRunTime) > intervalBetweenSeconds:
+				lastRunTime = timer()
+				val = getValue()
+				if shouldStopEvaluator(val):
+					return True, val
+		else:
+			if errorMessage:
+				raise RuntimeError(errorMessage)
+			return False, None
+
 	def _connectToRemoteServer(self):
 		"""Connects to the nvdaSpyServer
 		Because we do not know how far through the startup NVDA is, we have to poll
@@ -155,29 +200,17 @@ def quit_NVDA(self):
 
 	def assert_last_speech(self, expectedSpeech):
 		actualLastSpeech = self._runNvdaSpyKeyword("get_last_speech")
-		builtIn.should_be_equal_as_strings(
-			actualLastSpeech,
-			expectedSpeech,
-			msg="Actual speech != Expected speech"
-		)
+		_should_be_equal_as_strings(actualLastSpeech, expectedSpeech)
 
 	def assert_all_speech(self, expectedSpeech):
 		actualSpeech = self._runNvdaSpyKeyword("get_all_speech")
-		builtIn.should_be_equal_as_strings(
-			actualSpeech,
-			expectedSpeech,
-			msg="Actual speech != Expected speech",
-		)
+		_should_be_equal_as_strings(actualSpeech, expectedSpeech)
 
 	def assert_speech_since_index(self, index, expectedSpeech):
 		actualSpeech = self._runNvdaSpyKeyword("get_speech_since_index", index)
-		builtIn.should_be_equal_as_strings(
-			actualSpeech,
-			expectedSpeech,
-			msg="Actual speech != Expected speech",
-		)
+		_should_be_equal_as_strings(actualSpeech, expectedSpeech)
 
-	def wait_for_next_speech(self, maxWaitSeconds=5, raiseErrorOnTimeout=True):
+	def wait_for_next_speech(self, maxWaitSeconds=5.0, raiseErrorOnTimeout=True):
 		return self._blockUntilReturnsTrue(
 			func=lambda: self._runNvdaSpyKeyword("has_speech_occurred_since_last_check"),
 			giveUpAfterSeconds=maxWaitSeconds,
@@ -185,10 +218,10 @@ def wait_for_next_speech(self, maxWaitSeconds=5, raiseErrorOnTimeout=True):
 		)
 
 	def has_speech_finished(self):
-		speechOccurred = self.wait_for_next_speech(maxWaitSeconds=2, raiseErrorOnTimeout=False)
+		speechOccurred = self.wait_for_next_speech(maxWaitSeconds=0.5, raiseErrorOnTimeout=False)
 		return not speechOccurred
 
-	def wait_for_speech_to_finish(self, maxWaitSeconds=5):
+	def wait_for_speech_to_finish(self, maxWaitSeconds=5.0):
 		self._blockUntilReturnsTrue(
 			func=self.has_speech_finished,
 			giveUpAfterSeconds=maxWaitSeconds,
@@ -204,23 +237,23 @@ def reset_get_all_speech(self):
 
 	def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
 		sinceIndex = 0 if not sinceIndex else sinceIndex
-		def trueIfSpeechIndex():
-			index = self._runNvdaSpyKeyword("get_index_of_speech", speech, sinceIndex)
-			builtIn.log_to_console("value of index: ".format(index))
-			return index > 0
 
-		self._blockUntilReturnsTrue(
-			func=trueIfSpeechIndex,
+		success, speechIndex = self._blockUntilConditionMet(
+			getValue=lambda: self._runNvdaSpyKeyword("get_index_of_speech", speech, sinceIndex),
 			giveUpAfterSeconds=maxWaitSeconds,
+			shouldStopEvaluator=lambda speechIndex: speechIndex>=0,
+			intervalBetweenSeconds=0.5,
 			errorMessage="Specific speech did not occur before timeout: {}".format(speech)
 		)
+		return speechIndex
 
 	def wait_for_specific_speech_after_action(self, speech, action, *args):
 		self.wait_for_speech_to_finish()
 		index = self._runNvdaSpyKeyword("get_speech_index")
 		builtIn.run_keyword(action, *args)
-		self.wait_for_specific_speech(speech, index)
-		return self._runNvdaSpyKeyword("get_index_of_speech", speech, index)
+		index = self.wait_for_specific_speech(speech, index)
+		builtIn.log("got index: {}".format(index), level='INFO')
+		return index
 
 	def wait_for_any_speech_after_action(self, action, *args):
 		self.wait_for_speech_to_finish()
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index 9e90773111c..46c22eb2c7b 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -3,8 +3,9 @@
 import threading
 from robotremoteserver import RobotRemoteServer
 from logHandler import log
+whitespaceMinusSlashN = '\t\x0b\x0c\r '
 
-class SystemTestSpy:
+class SystemTestSpy(object):
 	def __init__(self):
 		self._nvdaStartupComplete = False
 		from core import postNvdaStartup
@@ -27,8 +28,9 @@ def _onNvdaSpeech(self, speechSequence=None):
 		self._nvdaSpeech.append(speechSequence)
 
 	def _getJoinedBaseStringsFromCommands(self, speechCommandArray):
-		baseStrings = [c.strip() for c in speechCommandArray if isinstance(c, basestring)]
-		return ' '.join(baseStrings).replace("  ", "\n").strip()
+		wsChars = whitespaceMinusSlashN
+		baseStrings = [c.strip(wsChars) for c in speechCommandArray if isinstance(c, basestring)]
+		return ''.join(baseStrings).strip()
 
 	# Start of Robot library API
 
@@ -46,12 +48,21 @@ def get_last_speech(self):
 	def get_all_speech(self):
 		return self.get_speech_since_index(self._allSpeechStartIndex)
 
+	def _flattenCommandsSeparatingWithNewline(self, commandArray):
+		f = [c for commands in commandArray for newlineJoined in [commands, [u"\n"]] for c in newlineJoined]
+		log.debug("f: {}".format(repr(f)))
+		return f
+
 	def get_speech_since_index(self, speechIndex):
-		speechCommands = [c for commands in self._nvdaSpeech[speechIndex:] for c in commands]
-		return self._getJoinedBaseStringsFromCommands(speechCommands)
+		speechCommands = self._flattenCommandsSeparatingWithNewline(
+			self._nvdaSpeech[speechIndex:]
+		)
+		joined = self._getJoinedBaseStringsFromCommands(speechCommands)
+		log.debug("joined: {}".format(joined))
+		return joined
 
 	def get_speech_index(self):
-		return len(self._nvdaSpeech) -1
+		return len(self._nvdaSpeech) - 1
 
 	def reset_all_speech_index(self):
 		self._allSpeechStartIndex = self.get_speech_index()
@@ -60,12 +71,12 @@ def get_index_of_speech(self, speech, indexHint=0):
 		log.debug("indexHint is: {}, speech is: {}".format(indexHint, speech))
 		for index, commands in enumerate(self._nvdaSpeech[indexHint:]):
 			index = index+indexHint
-			baseStrings = [c.strip().replace("  ", "\n") for c in commands if isinstance(c, basestring)]
+			baseStrings = [c.strip() for c in commands if isinstance(c, basestring)]
 			log.debug("baseStrings: \n{}".format(repr(baseStrings)))
 			if speech in baseStrings:
 				log.debug("at index: {}, Found: {}".format(index, speech))
 				return index
-		return 0
+		return -1
 
 
 
diff --git a/tests/system/variables.py b/tests/system/variables.py
index 90196ef1eb7..185819ac18f 100644
--- a/tests/system/variables.py
+++ b/tests/system/variables.py
@@ -1,23 +1,14 @@
 WELCOME_DIALOG_TEXT = (
-	"Welcome to NVDA\n"
-	"dialog\n"
-	"Welcome to NVDA!\n"
+	"Welcome to NVDA  dialog  Welcome to NVDA!\n"
 	"Most commands for controlling NVDA require you to hold down the NVDA key while pressing other keys.\n"
 	"By default, the numpad Insert and main Insert keys may both be used as the NVDA key.\n"
 	"You can also configure NVDA to use the CapsLock as the NVDA key.\n"
 	"Press NVDA+n at any time to activate the NVDA menu.\n"
-	"From this menu, you can configure NVDA, get help and access other NVDA functions. Options\n"
-	"grouping Keyboard layout:\n"
-	"combo box\n"
-	"desktop\n"
-	"collapsed\n"
-	"Alt+k"
+	"From this menu, you can configure NVDA, get help and access other NVDA functions.\n"
+	"Options  grouping\n"
+	"Keyboard layout:  combo box  desktop  collapsed  Alt+k"
 )
 QUIT_DIALOG_TEXT = (
-	"Exit NVDA\n"
-	"dialog What would you like to do?\n"
-	"combo box\n"
-	"Exit\n"
-	"collapsed\n"
-	"Alt+d\n"
+	"Exit NVDA  dialog\n"
+	"What would you like to do?  combo box  Exit  collapsed  Alt+d"
 )
\ No newline at end of file

From da0df66a4ceed79f612e93e26a398b519470e06b Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Sun, 1 Jul 2018 08:26:58 +0000
Subject: [PATCH 71/95] move spy lib logic into spy

---
 tests/system/initial.robot              |  14 +-
 tests/system/libraries/nvdaRobotLib.py  | 177 +++++---------------
 tests/system/libraries/systemTestSpy.py | 213 ++++++++++++++++++------
 tests/system/libraries/testutils.py     |  28 ++++
 4 files changed, 243 insertions(+), 189 deletions(-)
 create mode 100644 tests/system/libraries/testutils.py

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 51da756e82f..e75cf780317 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -16,14 +16,22 @@ Ensure NVDA runs at all
 	process should be running	nvdaAlias
 
 Ensure NVDA quits from keyboard
+	# TODO: can we make a better syntax for this? Maybe: 'join two spaces	sample text'
 	${Exit NVDA dialog} =	catenate	SEPARATOR=${SPACE * 2}	Exit NVDA	dialog
-	${INDEX} =	wait_for_specific_speech_after_action	${Exit NVDA dialog}	send key	insert	q
+	send key	insert	q
+	${INDEX} =	wait for specific speech	${Exit NVDA dialog}
+
 	wait for speech to finish
-	assert speech since index	${INDEX}	${QUIT_DIALOG_TEXT}
+	${actual speech} =	get speech from index until now	${INDEX}
+	assert strings are equal	${actual speech}	${QUIT_DIALOG_TEXT}
+
 	send key	enter
 	wait for process	nvdaAlias	timeout=5 sec
 
 Can read the welcome dialog
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
+	${Welcome dialog title} =	catenate	SEPARATOR=${SPACE * 2}	Welcome to NVDA	dialog
+	${INDEX} =	wait for specific speech	${Welcome dialog title}
 	wait for speech to finish
-	assert all speech	${WELCOME_DIALOG_TEXT}
\ No newline at end of file
+	${actual speech} =	get speech from index until now	${INDEX}
+	assert strings are equal	${actual speech}	${WELCOME_DIALOG_TEXT}
\ No newline at end of file
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 0ffbd8c8b32..fe6173d292d 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -1,11 +1,10 @@
 import os
 from os.path import join as pathJoin
 from os.path import abspath
-from timeit import default_timer as timer
 from robotremoteserver import test_remote_server, stop_remote_server
+from testutils import blockUntilConditionMet
 from robot.libraries.BuiltIn import BuiltIn
 
-
 builtIn = BuiltIn()
 process = builtIn.get_library_instance('Process')
 opSys = builtIn.get_library_instance('OperatingSystem')
@@ -20,28 +19,15 @@
 nvdaProfileWorkingDir = pathJoin(systemTestSourceDir, "nvdaProfile")
 nvdaSettingsSourceDir = pathJoin(systemTestSourceDir, "nvdaSettingsFiles")
 
+# TODO: find a better way to share this testutils code!!
+systemTestUtilsFileName = "testutils.py"
+systemTestUtilsSource = pathJoin(systemTestSourceDir, "libraries", systemTestUtilsFileName)
 systemTestSpyFileName = "systemTestSpy.py"
 systemTestSpySource = pathJoin(systemTestSourceDir, "libraries", systemTestSpyFileName)
 systemTestSpyInstallDir = pathJoin(nvdaProfileWorkingDir, "globalPlugins")
 systemTestSpyInstalled = pathJoin(systemTestSpyInstallDir, systemTestSpyFileName)
+systemTestUtilsInstalled = pathJoin(nvdaProfileWorkingDir, "systemTestLibs", systemTestUtilsFileName)
 
-def _should_be_equal_as_strings(actual, expected, ignore_case=False):
-	try:
-		builtIn.should_be_equal_as_strings(
-			actual,
-			expected,
-			msg="Actual speech != Expected speech",
-			ignore_case=ignore_case
-		)
-	except AssertionError:
-		builtIn.log(
-			"repr of actual vs expected (ignore_case={}):\n{}\nvs\n{}".format(
-				ignore_case,
-				repr(actual),
-				repr(expected)
-			)
-		)
-		raise
 
 class nvdaRobotLib(object):
 
@@ -50,6 +36,7 @@ def __init__(self):
 		self.nvdaHandle = None
 
 	def setup_nvda_profile(self, settingsFileName):
+		builtIn.log("Copying files into NVDA profile")
 		opSys.create_directory(systemTestSpyInstallDir)
 		opSys.copy_file(systemTestSpySource, systemTestSpyInstallDir)
 		opSys.copy_file(
@@ -58,6 +45,8 @@ def setup_nvda_profile(self, settingsFileName):
 		)
 
 	def teardown_nvda_profile(self):
+		builtIn.log("Removing files from NVDA profile")
+		# TODO: probably dont need to remove the raw python file?
 		opSys.remove_file(systemTestSpyInstalled)
 		opSys.remove_file(systemTestSpyInstalled+"c")  # also remove the .pyc
 		opSys.remove_file(
@@ -76,51 +65,6 @@ def _startNVDAProcess(self):
 		)
 		return handle
 
-	def _blockUntilReturnsTrue(self, func, giveUpAfterSeconds, intervalBetweenSeconds=0.1, errorMessage=None):
-		"""Call 'func' every 'intervalBetweenSeconds' seconds until 'func' returns True until
-		'giveUpAfterSeconds' is reached.
-		If 'func' does not return true and 'errorMessage' is provided, a RuntimeError is raised
-		using the provided errorMessage.
-		@return True if 'func' returns True before 'giveUpAfterSeconds' is reached, else False
-		"""
-		startTime = timer()
-		lastRunTime = startTime - intervalBetweenSeconds+1  # ensure we start trying immediately
-		while (timer() - startTime) < giveUpAfterSeconds:
-			if (timer() - lastRunTime) > intervalBetweenSeconds:
-				lastRunTime = timer()
-				if func():
-					return True
-		else:
-			if errorMessage:
-				raise RuntimeError(errorMessage)
-			return False
-
-
-	def _blockUntilConditionMet(
-			self,
-			getValue,
-			giveUpAfterSeconds,
-			shouldStopEvaluator=lambda value: bool(value),
-			intervalBetweenSeconds=0.1,
-			errorMessage=None):
-		"""Repeatedly tries to get a value up until a time limit expires. Tries are separated by
-		a time interval. The call will block until shouldStopEvaluator returns True when given the value,
-		the default evaluator just returns the value converted to a boolean.
-		@return A tuple, (True, value) if evaluator condition is met, otherwise (False, None)
-		@raises RuntimeError if the time limit expires and an errorMessage is given.
-		"""
-		startTime = timer()
-		lastRunTime = startTime - intervalBetweenSeconds+1  # ensure we start trying immediately
-		while (timer() - startTime) < giveUpAfterSeconds:
-			if (timer() - lastRunTime) > intervalBetweenSeconds:
-				lastRunTime = timer()
-				val = getValue()
-				if shouldStopEvaluator(val):
-					return True, val
-		else:
-			if errorMessage:
-				raise RuntimeError(errorMessage)
-			return False, None
 
 	def _connectToRemoteServer(self):
 		"""Connects to the nvdaSpyServer
@@ -131,29 +75,35 @@ def _connectToRemoteServer(self):
 				No connection could be made because the target machine actively refused it"
 		Instead we wait until the remote server is available before importing the library and continuing.
 		"""
+
+		builtIn.log("Waiting for nvdaSpy to be available at: {}".format(spyServerURI))
 		# Importing the 'Remote' library always succeeds, even when a connection can not be made.
 		# If that happens, then some 'Remote' keyword will fail at some later point.
 		# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.
-		self._blockUntilReturnsTrue(
-			func=lambda: test_remote_server(spyServerURI, log=False),
+		blockUntilConditionMet(
+			getValue=lambda: test_remote_server(spyServerURI, log=False),
 			giveUpAfterSeconds=10,
 			errorMessage="Unable to connect to nvdaSpy",
 		)
-
+		builtIn.log("Connecting to nvdaSpy")
+		maxRemoteKeywordDurationSeconds = 30  # If any remote call takes longer than this, the connection will be closed!
 		builtIn.import_library(
 			"Remote",  # name of library to import
 			# Arguments to construct the library instance:
 			"uri={}".format(spyServerURI),
-			"timeout=2",  # seconds
+			"timeout={}".format(maxRemoteKeywordDurationSeconds),
 			# Set an alias for the imported library instance
 			"WITH NAME",
 			"nvdaSpy",
 		)
+		builtIn.log("Getting nvdaSpy library instance")
 		self.nvdaSpy = builtIn.get_library_instance(spyAlias)
+		self._runNvdaSpyKeyword("set_max_keyword_duration", maxSeconds=maxRemoteKeywordDurationSeconds)
 
 	def _runNvdaSpyKeyword(self, keyword, *args, **kwargs):
 		if not args: args = []
 		if not kwargs: kwargs = {}
+		builtIn.log("nvdaSpy keyword: {} args: {}, kwargs: {}".format(keyword, args, kwargs))
 		return self.nvdaSpy.run_keyword(keyword, args, kwargs)
 
 	def start_NVDA(self, settingsFileName):
@@ -161,18 +111,12 @@ def start_NVDA(self, settingsFileName):
 		nvdaProcessHandle = self._startNVDAProcess()
 		process.process_should_be_running(nvdaProcessHandle)
 		self._connectToRemoteServer()
-		self.wait_for_NVDA_startup_to_complete()
+		self._runNvdaSpyKeyword("wait_for_NVDA_startup_to_complete")
 		return nvdaProcessHandle
 
-	def wait_for_NVDA_startup_to_complete(self):
-		self._blockUntilReturnsTrue(
-			func=lambda: self._runNvdaSpyKeyword("is_NVDA_startup_complete"),
-			giveUpAfterSeconds=10,
-			errorMessage="Unable to connect to nvdaSpy",
-		)
-
 	def save_NVDA_log(self):
 		"""NVDA logs are saved to the ${OUTPUT DIR}/nvdaTestRunLogs/${SUITE NAME}-${TEST NAME}-nvda.log"""
+		builtIn.log("saving NVDA log")
 		outDir = builtIn.get_variable_value("${OUTPUT DIR}", )
 		suiteName = builtIn.get_variable_value("${SUITE NAME}")
 		testName = builtIn.get_variable_value("${TEST NAME}")
@@ -187,7 +131,7 @@ def save_NVDA_log(self):
 		)
 
 	def quit_NVDA(self):
-		stop_remote_server(spyServerURI, log=False)
+		builtIn.log("Stopping nvdaSpy server: {}".format(spyServerURI))
 		# remove the spy so that if nvda is run manually against this config it does not interfere.
 		self.teardown_nvda_profile()
 		process.run_process(
@@ -198,64 +142,21 @@ def quit_NVDA(self):
 		process.wait_for_process(self.nvdaHandle)
 		self.save_NVDA_log()
 
-	def assert_last_speech(self, expectedSpeech):
-		actualLastSpeech = self._runNvdaSpyKeyword("get_last_speech")
-		_should_be_equal_as_strings(actualLastSpeech, expectedSpeech)
-
-	def assert_all_speech(self, expectedSpeech):
-		actualSpeech = self._runNvdaSpyKeyword("get_all_speech")
-		_should_be_equal_as_strings(actualSpeech, expectedSpeech)
-
-	def assert_speech_since_index(self, index, expectedSpeech):
-		actualSpeech = self._runNvdaSpyKeyword("get_speech_since_index", index)
-		_should_be_equal_as_strings(actualSpeech, expectedSpeech)
-
-	def wait_for_next_speech(self, maxWaitSeconds=5.0, raiseErrorOnTimeout=True):
-		return self._blockUntilReturnsTrue(
-			func=lambda: self._runNvdaSpyKeyword("has_speech_occurred_since_last_check"),
-			giveUpAfterSeconds=maxWaitSeconds,
-			errorMessage="Speech did not start before timeout" if raiseErrorOnTimeout else None
-		)
-
-	def has_speech_finished(self):
-		speechOccurred = self.wait_for_next_speech(maxWaitSeconds=0.5, raiseErrorOnTimeout=False)
-		return not speechOccurred
-
-	def wait_for_speech_to_finish(self, maxWaitSeconds=5.0):
-		self._blockUntilReturnsTrue(
-			func=self.has_speech_finished,
-			giveUpAfterSeconds=maxWaitSeconds,
-			errorMessage="Speech did not finish before timeout"
-		)
-
-	def wait_for_speech_to_start_and_finish(self):
-		self.wait_for_next_speech()
-		self.wait_for_speech_to_finish()
-
-	def reset_get_all_speech(self):
-		self._runNvdaSpyKeyword("reset_all_speech_index")
-
-	def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
-		sinceIndex = 0 if not sinceIndex else sinceIndex
-
-		success, speechIndex = self._blockUntilConditionMet(
-			getValue=lambda: self._runNvdaSpyKeyword("get_index_of_speech", speech, sinceIndex),
-			giveUpAfterSeconds=maxWaitSeconds,
-			shouldStopEvaluator=lambda speechIndex: speechIndex>=0,
-			intervalBetweenSeconds=0.5,
-			errorMessage="Specific speech did not occur before timeout: {}".format(speech)
-		)
-		return speechIndex
-
-	def wait_for_specific_speech_after_action(self, speech, action, *args):
-		self.wait_for_speech_to_finish()
-		index = self._runNvdaSpyKeyword("get_speech_index")
-		builtIn.run_keyword(action, *args)
-		index = self.wait_for_specific_speech(speech, index)
-		builtIn.log("got index: {}".format(index), level='INFO')
-		return index
-
-	def wait_for_any_speech_after_action(self, action, *args):
-		self.wait_for_speech_to_finish()
-		builtIn.run_keyword(action, *args)
-		self.wait_for_next_speech()
\ No newline at end of file
+# TODO: this shouldn't be a member function, but it is not available from robot if it is not??
+	def assert_strings_are_equal(self, actual, expected, ignore_case=False):
+		try:
+			builtIn.should_be_equal_as_strings(
+				actual,
+				expected,
+				msg="Actual speech != Expected speech",
+				ignore_case=ignore_case
+			)
+		except AssertionError:
+			builtIn.log(
+				"repr of actual vs expected (ignore_case={}):\n{}\nvs\n{}".format(
+					ignore_case,
+					repr(actual),
+					repr(expected)
+				)
+			)
+			raise
\ No newline at end of file
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index 46c22eb2c7b..27485cc7f0d 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -1,84 +1,198 @@
 import globalPluginHandler
-import signal
 import threading
 from robotremoteserver import RobotRemoteServer
 from logHandler import log
+from timeit import default_timer as timer
 whitespaceMinusSlashN = '\t\x0b\x0c\r '
 
+
+def blockUntilConditionMet(
+		getValue,
+		giveUpAfterSeconds,
+		shouldStopEvaluator=lambda value: bool(value),
+		intervalBetweenSeconds=0.1,
+		errorMessage=None):
+	"""Repeatedly tries to get a value up until a time limit expires. Tries are separated by
+	a time interval. The call will block until shouldStopEvaluator returns True when given the value,
+	the default evaluator just returns the value converted to a boolean.
+	@return A tuple, (True, value) if evaluator condition is met, otherwise (False, None)
+	@raises RuntimeError if the time limit expires and an errorMessage is given.
+	"""
+	assert callable(getValue)
+	assert callable(shouldStopEvaluator)
+	startTime = timer()
+	lastRunTime = startTime
+	firstRun = True  # ensure we start immediately
+	while (timer() - startTime) < giveUpAfterSeconds:
+		if firstRun or (timer() - lastRunTime) > intervalBetweenSeconds:
+			firstRun = False
+			lastRunTime = timer()
+			val = getValue()
+			if shouldStopEvaluator(val):
+				return True, val
+	else:
+		if errorMessage:
+			raise AssertionError(errorMessage)
+		return False, None
+
+
 class SystemTestSpy(object):
+	SPEECH_HAS_FINISHED_SECONDS = 0.5
+
 	def __init__(self):
-		self._nvdaStartupComplete = False
-		from core import postNvdaStartup
-		postNvdaStartup.register(self._onNvdaStartupComplete)
-		from speech import preSpeech
-		preSpeech.register(self._onNvdaSpeech)
 		self._nvdaSpeech = [
 			[""],  # initialise with an empty string, this allows for access via [-1]. This is equiv to no speech.
 		]
 		self._allSpeechStartIndex = 0
 		self._speechOccurred = False
+		self.isNvdaStartupComplete = False
+		self.lastSpeechTime = timer()
+		self._registerWithExtensionPoints()
+
+	def _registerWithExtensionPoints(self):
+		from core import postNvdaStartup
+		from speech import preSpeech
+		postNvdaStartup.register(self._onNvdaStartupComplete)
+		preSpeech.register(self._onNvdaSpeech)
 
+	# callbacks for extension points
 	def _onNvdaStartupComplete(self):
-		self._nvdaStartupComplete = True
-		self.reset_all_speech_index()
+		self.isNvdaStartupComplete = True
 
 	def _onNvdaSpeech(self, speechSequence=None):
 		if not speechSequence: return
-		self._speechOccurred = True
-		self._nvdaSpeech.append(speechSequence)
+		with threading.Lock():
+			self._speechOccurred = True
+			self.lastSpeechTime = timer()
+			self._nvdaSpeech.append(speechSequence)
+
+	# Private helper methods
+	def _flattenCommandsSeparatingWithNewline(self, commandArray):
+		f = [c for commands in commandArray for newlineJoined in [commands, [u"\n"]] for c in newlineJoined]
+		return f
 
 	def _getJoinedBaseStringsFromCommands(self, speechCommandArray):
 		wsChars = whitespaceMinusSlashN
 		baseStrings = [c.strip(wsChars) for c in speechCommandArray if isinstance(c, basestring)]
 		return ''.join(baseStrings).strip()
 
-	# Start of Robot library API
-
-	def is_NVDA_startup_complete(self):
-		return self._nvdaStartupComplete
-
-	def has_speech_occurred_since_last_check(self):
-		speechOccurred = self._speechOccurred
-		self._speechOccurred = False
+	# Public methods
+	def checkIfSpeechOccurredAndReset(self):
+		# don't let _speechOccurred get updated and overwritten with False
+		with threading.Lock():
+			speechOccurred = self._speechOccurred
+			self._speechOccurred = False
 		return speechOccurred
 
-	def get_last_speech(self):
-		return self._getJoinedBaseStringsFromCommands(self._nvdaSpeech[-1])
+	def getSpeechAtIndex(self, speechIndex):
+		with threading.Lock():
+			return self._getJoinedBaseStringsFromCommands(self._nvdaSpeech[speechIndex])
+
+	def getSpeechSinceIndex(self, speechIndex):
+		with threading.Lock():
+			speechCommands = self._flattenCommandsSeparatingWithNewline(
+				self._nvdaSpeech[speechIndex:]
+			)
+			joined = self._getJoinedBaseStringsFromCommands(speechCommands)
+			return joined
+
+	def getIndexOfLastSpeech(self):
+		with threading.Lock():
+			return len(self._nvdaSpeech) - 1
+
+	def getIndexOfSpeech(self, speech, indexHint=0):
+		with threading.Lock():
+			log.debug("from index:{}, looking for speech: {}".format(indexHint, speech))
+			for index, commands in enumerate(self._nvdaSpeech[indexHint:]):
+				index = index+indexHint
+				baseStrings = [c.strip() for c in commands if isinstance(c, basestring)]
+				if any(speech in x for x in baseStrings):
+					log.debug("found it")
+					return index
+			log.debug("did not find it")
+			return -1
+
+	def hasSpeechFinished(self):
+		return self.SPEECH_HAS_FINISHED_SECONDS < timer() - self.lastSpeechTime
+
+	def dumpSpeechToLog(self):
+		log.debug("All speech:\n{}".format(repr(self._nvdaSpeech)))
+
+class NvdaSpyLib(object):
+	_spy = None  # type: SystemTestSpy
+
+	def __init__(self, systemTestSpy):
+		self._spy = systemTestSpy
+		self._allSpeechStartIndex = self._spy.getIndexOfLastSpeech()
+		self._maxKeywordDuration=30
+
+	def _minTimeout(self, timeout):
+		"""Helper to get the minimum value, the timeout passed in, or self._maxKeywordDuration"""
+		return min(timeout, self._maxKeywordDuration)
 
-	def get_all_speech(self):
-		return self.get_speech_since_index(self._allSpeechStartIndex)
+	# Start of Robot library API
 
-	def _flattenCommandsSeparatingWithNewline(self, commandArray):
-		f = [c for commands in commandArray for newlineJoined in [commands, [u"\n"]] for c in newlineJoined]
-		log.debug("f: {}".format(repr(f)))
-		return f
+	def set_max_keyword_duration(self, maxSeconds):
+		"""This should only be called after importing the library, and should match the 'timeout' value given to the
+		robot.libraries.Remote instance"""
+		self._maxKeywordDuration = maxSeconds-1
 
-	def get_speech_since_index(self, speechIndex):
-		speechCommands = self._flattenCommandsSeparatingWithNewline(
-			self._nvdaSpeech[speechIndex:]
+	def wait_for_NVDA_startup_to_complete(self):
+		blockUntilConditionMet(
+			getValue=lambda: self._spy.isNvdaStartupComplete,
+			giveUpAfterSeconds=self._minTimeout(10),
+			errorMessage="Unable to connect to nvdaSpy",
 		)
-		joined = self._getJoinedBaseStringsFromCommands(speechCommands)
-		log.debug("joined: {}".format(joined))
-		return joined
+		if self._spy.isNvdaStartupComplete:
+			self.reset_all_speech_index()
 
-	def get_speech_index(self):
-		return len(self._nvdaSpeech) - 1
-
-	def reset_all_speech_index(self):
-		self._allSpeechStartIndex = self.get_speech_index()
+	def get_last_speech(self):
+		return self._spy.getSpeechAtIndex(-1)
 
-	def get_index_of_speech(self, speech, indexHint=0):
-		log.debug("indexHint is: {}, speech is: {}".format(indexHint, speech))
-		for index, commands in enumerate(self._nvdaSpeech[indexHint:]):
-			index = index+indexHint
-			baseStrings = [c.strip() for c in commands if isinstance(c, basestring)]
-			log.debug("baseStrings: \n{}".format(repr(baseStrings)))
-			if speech in baseStrings:
-				log.debug("at index: {}, Found: {}".format(index, speech))
-				return index
-		return -1
+	def get_all_speech(self):
+		return self._spy.getSpeechSinceIndex(self._allSpeechStartIndex)
 
+	def get_speech_from_index_until_now(self, speechIndex):
+		return self._spy.getSpeechSinceIndex(speechIndex)
 
+	def reset_all_speech_index(self):
+		self._allSpeechStartIndex = self._spy.getIndexOfLastSpeech()
+		return self._allSpeechStartIndex
+
+	def get_last_speech_index(self):
+		return self._spy.getIndexOfLastSpeech()
+
+# TODO: does this need to return the index of this speech? What if the "next speech" has already past when this is called?
+	def wait_for_speech_to_start(self, maxWaitSeconds=5.0, raiseErrorOnTimeout=True):
+		useErrorMesg = "Speech did not start before timeout" if raiseErrorOnTimeout else None
+		success, value = blockUntilConditionMet(
+			getValue=lambda: self._spy.checkIfSpeechOccurredAndReset(),
+			giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
+			errorMessage=useErrorMesg
+		)
+		return success
+
+	def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
+		sinceIndex = 0 if not sinceIndex else sinceIndex
+		try:
+			success, speechIndex = blockUntilConditionMet(
+				getValue=lambda: self._spy.getIndexOfSpeech(speech, sinceIndex),
+				giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
+				shouldStopEvaluator=lambda speechIndex: speechIndex >= 0,
+				intervalBetweenSeconds=0.1,
+				errorMessage="Specific speech did not occur before timeout: {}".format(speech)
+			)
+		except AssertionError:
+			self._spy.dumpSpeechToLog()
+			raise
+		return speechIndex
+
+	def wait_for_speech_to_finish(self, maxWaitSeconds=5.0):
+		blockUntilConditionMet(
+			getValue=self._spy.hasSpeechFinished,
+			giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
+			errorMessage="Speech did not finish before timeout"
+		)
 
 class SystemTestSpyServer(object):
 	def __init__(self):
@@ -86,8 +200,9 @@ def __init__(self):
 
 	def start(self):
 		log.debug("TestSpyPlugin started")
+		spy = SystemTestSpy()  # spies on NVDA
 		server = self._server = RobotRemoteServer(
-			SystemTestSpy(),  # provides actual spy behaviour
+			NvdaSpyLib(spy),  # provides library behaviour
 			port=8270,  # default:8270 is `registered by IANA` for remote server usage. Two ASCII values, RF.
 			serve=False  # we want to start this serving on another thread so as not to block.
 		)
@@ -96,8 +211,10 @@ def start(self):
 		server_thread.start()
 
 	def stop(self):
+		log.debug("Stop SystemTestSpyServer called")
 		self._server.stop()
 
+
 class GlobalPlugin(globalPluginHandler.GlobalPlugin):
 
 	def __init__(self):
diff --git a/tests/system/libraries/testutils.py b/tests/system/libraries/testutils.py
new file mode 100644
index 00000000000..cf591d4aafd
--- /dev/null
+++ b/tests/system/libraries/testutils.py
@@ -0,0 +1,28 @@
+from timeit import default_timer as timer
+
+def blockUntilConditionMet(
+		getValue,
+		giveUpAfterSeconds,
+		shouldStopEvaluator=lambda value: bool(value),
+		intervalBetweenSeconds=0.1,
+		errorMessage=None):
+	"""Repeatedly tries to get a value up until a time limit expires. Tries are separated by
+	a time interval. The call will block until shouldStopEvaluator returns True when given the value,
+	the default evaluator just returns the value converted to a boolean.
+	@return A tuple, (True, value) if evaluator condition is met, otherwise (False, None)
+	@raises RuntimeError if the time limit expires and an errorMessage is given.
+	"""
+	startTime = timer()
+	lastRunTime = startTime
+	firstRun=True  # ensure we start trying immediately
+	while (timer() - startTime) < giveUpAfterSeconds:
+		if firstRun or (timer() - lastRunTime) > intervalBetweenSeconds:
+			firstRun = False
+			lastRunTime = timer()
+			val = getValue()
+			if shouldStopEvaluator(val):
+				return True, val
+	else:
+		if errorMessage:
+			raise AssertionError(errorMessage)
+		return False, None

From bc869e6ff3addc5dee207548ad31ee0b0dc68f3e Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Mon, 2 Jul 2018 02:23:09 +0000
Subject: [PATCH 72/95] Install systemTestSpy as a package on setup

---
 tests/system/libraries/nvdaRobotLib.py  | 48 +++++++++++++------------
 tests/system/libraries/systemTestSpy.py | 31 +---------------
 tests/system/libraries/testutils.py     |  4 ++-
 3 files changed, 30 insertions(+), 53 deletions(-)

diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index fe6173d292d..0c9b8ae6159 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -4,29 +4,23 @@
 from robotremoteserver import test_remote_server, stop_remote_server
 from testutils import blockUntilConditionMet
 from robot.libraries.BuiltIn import BuiltIn
+from robot.libraries.OperatingSystem import OperatingSystem
+from robot.libraries.Process import Process
 
-builtIn = BuiltIn()
-process = builtIn.get_library_instance('Process')
-opSys = builtIn.get_library_instance('OperatingSystem')
+builtIn = BuiltIn()  # type: BuiltIn
+process = builtIn.get_library_instance('Process')  # type: Process
+opSys = builtIn.get_library_instance('OperatingSystem')  # type: OperatingSystem
 
 spyServerPort = 8270  # is `registered by IANA` for remote server usage. Two ASCII values:'RF'
 spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
 spyAlias = "nvdaSpy"
 
+# Paths
 nvdaLogFilePath = abspath("source/nvda.log")
-
 systemTestSourceDir = abspath("tests/system")
 nvdaProfileWorkingDir = pathJoin(systemTestSourceDir, "nvdaProfile")
-nvdaSettingsSourceDir = pathJoin(systemTestSourceDir, "nvdaSettingsFiles")
-
-# TODO: find a better way to share this testutils code!!
-systemTestUtilsFileName = "testutils.py"
-systemTestUtilsSource = pathJoin(systemTestSourceDir, "libraries", systemTestUtilsFileName)
-systemTestSpyFileName = "systemTestSpy.py"
-systemTestSpySource = pathJoin(systemTestSourceDir, "libraries", systemTestSpyFileName)
-systemTestSpyInstallDir = pathJoin(nvdaProfileWorkingDir, "globalPlugins")
-systemTestSpyInstalled = pathJoin(systemTestSpyInstallDir, systemTestSpyFileName)
-systemTestUtilsInstalled = pathJoin(nvdaProfileWorkingDir, "systemTestLibs", systemTestUtilsFileName)
+profileGlobalPluginsDir = pathJoin(nvdaProfileWorkingDir, "globalPlugins")
+profileSysTestSpyPackageDir = pathJoin(profileGlobalPluginsDir, "systemTestSpy")
 
 
 class nvdaRobotLib(object):
@@ -37,20 +31,30 @@ def __init__(self):
 
 	def setup_nvda_profile(self, settingsFileName):
 		builtIn.log("Copying files into NVDA profile")
-		opSys.create_directory(systemTestSpyInstallDir)
-		opSys.copy_file(systemTestSpySource, systemTestSpyInstallDir)
 		opSys.copy_file(
-			os.path.join(nvdaSettingsSourceDir, settingsFileName),
-			os.path.join(nvdaProfileWorkingDir, "nvda.ini")
+			pathJoin(systemTestSourceDir, "nvdaSettingsFiles", settingsFileName),
+			pathJoin(nvdaProfileWorkingDir, "nvda.ini")
+		)
+		# create a package to use as the globalPlugin
+		opSys.create_directory(profileSysTestSpyPackageDir)
+		opSys.copy_file(
+			pathJoin(systemTestSourceDir, "libraries", "systemTestSpy.py"),
+			pathJoin(profileSysTestSpyPackageDir, "__init__.py")
+		)
+		testUtilsFileName = "testutils.py"
+		opSys.copy_file(
+			pathJoin(systemTestSourceDir, "libraries", testUtilsFileName),
+			pathJoin(profileSysTestSpyPackageDir, testUtilsFileName)
 		)
 
 	def teardown_nvda_profile(self):
 		builtIn.log("Removing files from NVDA profile")
-		# TODO: probably dont need to remove the raw python file?
-		opSys.remove_file(systemTestSpyInstalled)
-		opSys.remove_file(systemTestSpyInstalled+"c")  # also remove the .pyc
 		opSys.remove_file(
-			os.path.join(nvdaProfileWorkingDir, "nvda.ini")
+			pathJoin(nvdaProfileWorkingDir, "nvda.ini")
+		)
+		opSys.remove_directory(
+			profileSysTestSpyPackageDir,
+			recursive=True
 		)
 
 	def _startNVDAProcess(self):
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index 27485cc7f0d..c419769990f 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -1,41 +1,12 @@
 import globalPluginHandler
 import threading
+from testutils import blockUntilConditionMet
 from robotremoteserver import RobotRemoteServer
 from logHandler import log
 from timeit import default_timer as timer
 whitespaceMinusSlashN = '\t\x0b\x0c\r '
 
 
-def blockUntilConditionMet(
-		getValue,
-		giveUpAfterSeconds,
-		shouldStopEvaluator=lambda value: bool(value),
-		intervalBetweenSeconds=0.1,
-		errorMessage=None):
-	"""Repeatedly tries to get a value up until a time limit expires. Tries are separated by
-	a time interval. The call will block until shouldStopEvaluator returns True when given the value,
-	the default evaluator just returns the value converted to a boolean.
-	@return A tuple, (True, value) if evaluator condition is met, otherwise (False, None)
-	@raises RuntimeError if the time limit expires and an errorMessage is given.
-	"""
-	assert callable(getValue)
-	assert callable(shouldStopEvaluator)
-	startTime = timer()
-	lastRunTime = startTime
-	firstRun = True  # ensure we start immediately
-	while (timer() - startTime) < giveUpAfterSeconds:
-		if firstRun or (timer() - lastRunTime) > intervalBetweenSeconds:
-			firstRun = False
-			lastRunTime = timer()
-			val = getValue()
-			if shouldStopEvaluator(val):
-				return True, val
-	else:
-		if errorMessage:
-			raise AssertionError(errorMessage)
-		return False, None
-
-
 class SystemTestSpy(object):
 	SPEECH_HAS_FINISHED_SECONDS = 0.5
 
diff --git a/tests/system/libraries/testutils.py b/tests/system/libraries/testutils.py
index cf591d4aafd..748d70c56fe 100644
--- a/tests/system/libraries/testutils.py
+++ b/tests/system/libraries/testutils.py
@@ -12,9 +12,11 @@ def blockUntilConditionMet(
 	@return A tuple, (True, value) if evaluator condition is met, otherwise (False, None)
 	@raises RuntimeError if the time limit expires and an errorMessage is given.
 	"""
+	assert callable(getValue)
+	assert callable(shouldStopEvaluator)
 	startTime = timer()
 	lastRunTime = startTime
-	firstRun=True  # ensure we start trying immediately
+	firstRun = True  # ensure we start immediately
 	while (timer() - startTime) < giveUpAfterSeconds:
 		if firstRun or (timer() - lastRunTime) > intervalBetweenSeconds:
 			firstRun = False

From f20dc359b0e2d8180dee89a159684c72905158fe Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Mon, 2 Jul 2018 02:23:52 +0000
Subject: [PATCH 73/95] Cleanup files, add copyright headers

---
 tests/system/initial.robot              |  4 ++
 tests/system/libraries/nvdaRobotLib.py  | 11 +++++-
 tests/system/libraries/sendKey.py       | 50 +++++--------------------
 tests/system/libraries/systemTestSpy.py | 10 +++++
 tests/system/libraries/testutils.py     | 11 ++++++
 5 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index e75cf780317..957f259ead5 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -1,3 +1,7 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
 *** Settings ***
 Documentation	Basic start and exit tests
 
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 0c9b8ae6159..f7a65c9b756 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -1,4 +1,13 @@
-import os
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
+
+"""This file provides robot library functions for NVDA system tests. It contains helper methods for system tests,
+most specifically related to the setup for, starting of, quiting of, and cleanup of, NVDA. This is in contrast with
+the systemTestSpy.py file, which provides library functions related to monitoring / asserting NVDA output.
+"""
+
 from os.path import join as pathJoin
 from os.path import abspath
 from robotremoteserver import test_remote_server, stop_remote_server
diff --git a/tests/system/libraries/sendKey.py b/tests/system/libraries/sendKey.py
index d3f3351ef99..807009abfe5 100644
--- a/tests/system/libraries/sendKey.py
+++ b/tests/system/libraries/sendKey.py
@@ -1,45 +1,15 @@
-import time
-import pyautogui
-pyautogui.FAILSAFE=False
-import ctypes
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
 
+"""This file provides system test library functions for sending keyboard key presses.
 """
-['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(',
-')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7',
-'8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`',
-'a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
-'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~',
-'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace',
-'browserback', 'browserfavorites', 'browserforward', 'browserhome',
-'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear',
-'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete',
-'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10',
-'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20',
-'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9',
-'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja',
-'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail',
-'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack',
-'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6',
-'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn',
-'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn',
-'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator',
-'shift', 'shiftleft', 'shiftright', 'sleep', 'space', 'stop', 'subtract', 'tab',
-'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen',
-'command', 'option', 'optionleft', 'optionright']
-"""
+import pyautogui
+pyautogui.FAILSAFE = False
 
 def send_key(*keys):
+	"""Sends the keys as if pressed by the user.
+	Full list of keys: pyautogui.KEYBOARD_KEY
+	"""
 	pyautogui.hotkey(*keys)
-
-def get_foreground_name():
-	hwnd=ctypes.windll.user32.GetForegroundWindow()
-	buf=ctypes.create_unicode_buffer(1024)
-	ctypes.windll.user32.InternalGetWindowText(hwnd,buf,1023)
-	return buf.value
-
-def wait_for_foreground(name,timeout=5):
-	endTime=time.time()+timeout
-	while get_foreground_name()!=name:
-		time.sleep(0.25)
-		if time.time()>endTime:
-			raise RuntimeError("Timeout")
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index c419769990f..e1573d49d96 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -1,3 +1,13 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
+
+"""This file provides spy and robot library behaviour for NVDA system tests.
+It is copied into the (system test specific) NVDA profile directory. It becomes the '__init__.py' file as part of a
+package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library.
+"""
+
 import globalPluginHandler
 import threading
 from testutils import blockUntilConditionMet
diff --git a/tests/system/libraries/testutils.py b/tests/system/libraries/testutils.py
index 748d70c56fe..657a83af95d 100644
--- a/tests/system/libraries/testutils.py
+++ b/tests/system/libraries/testutils.py
@@ -1,5 +1,16 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
+
+"""This file provides utility methods for NVDA system tests.
+It is copied into the (system test specific) NVDA profile directory as part of a
+package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library.
+"""
+
 from timeit import default_timer as timer
 
+
 def blockUntilConditionMet(
 		getValue,
 		giveUpAfterSeconds,

From 612659dcf88c683b98417fe3ccd8c35bd251e78e Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Mon, 2 Jul 2018 04:36:51 +0000
Subject: [PATCH 74/95] Use _ to prevent imports from leaking as keywords

---
 tests/system/libraries/nvdaRobotLib.py  | 43 +++++++++++++------------
 tests/system/libraries/systemTestSpy.py | 19 +++++------
 tests/system/libraries/testutils.py     | 12 +++----
 3 files changed, 38 insertions(+), 36 deletions(-)

diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index f7a65c9b756..f7e3fe7a817 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -7,11 +7,11 @@
 most specifically related to the setup for, starting of, quiting of, and cleanup of, NVDA. This is in contrast with
 the systemTestSpy.py file, which provides library functions related to monitoring / asserting NVDA output.
 """
-
-from os.path import join as pathJoin
-from os.path import abspath
-from robotremoteserver import test_remote_server, stop_remote_server
-from testutils import blockUntilConditionMet
+# imported methods start with underscore (_) so they don't get imported into robot files as keywords
+from os.path import join as _pJoin
+from os.path import abspath as _abspath
+from robotremoteserver import test_remote_server as _testRemoteServer, stop_remote_server as _stopRemoteServer
+from testutils import _blockUntilConditionMet
 from robot.libraries.BuiltIn import BuiltIn
 from robot.libraries.OperatingSystem import OperatingSystem
 from robot.libraries.Process import Process
@@ -25,11 +25,11 @@
 spyAlias = "nvdaSpy"
 
 # Paths
-nvdaLogFilePath = abspath("source/nvda.log")
-systemTestSourceDir = abspath("tests/system")
-nvdaProfileWorkingDir = pathJoin(systemTestSourceDir, "nvdaProfile")
-profileGlobalPluginsDir = pathJoin(nvdaProfileWorkingDir, "globalPlugins")
-profileSysTestSpyPackageDir = pathJoin(profileGlobalPluginsDir, "systemTestSpy")
+nvdaLogFilePath = _abspath("source/nvda.log")
+systemTestSourceDir = _abspath("tests/system")
+nvdaProfileWorkingDir = _pJoin(systemTestSourceDir, "nvdaProfile")
+profileGlobalPluginsDir = _pJoin(nvdaProfileWorkingDir, "globalPlugins")
+profileSysTestSpyPackageDir = _pJoin(profileGlobalPluginsDir, "systemTestSpy")
 
 
 class nvdaRobotLib(object):
@@ -41,25 +41,25 @@ def __init__(self):
 	def setup_nvda_profile(self, settingsFileName):
 		builtIn.log("Copying files into NVDA profile")
 		opSys.copy_file(
-			pathJoin(systemTestSourceDir, "nvdaSettingsFiles", settingsFileName),
-			pathJoin(nvdaProfileWorkingDir, "nvda.ini")
+			_pJoin(systemTestSourceDir, "nvdaSettingsFiles", settingsFileName),
+			_pJoin(nvdaProfileWorkingDir, "nvda.ini")
 		)
 		# create a package to use as the globalPlugin
 		opSys.create_directory(profileSysTestSpyPackageDir)
 		opSys.copy_file(
-			pathJoin(systemTestSourceDir, "libraries", "systemTestSpy.py"),
-			pathJoin(profileSysTestSpyPackageDir, "__init__.py")
+			_pJoin(systemTestSourceDir, "libraries", "systemTestSpy.py"),
+			_pJoin(profileSysTestSpyPackageDir, "__init__.py")
 		)
 		testUtilsFileName = "testutils.py"
 		opSys.copy_file(
-			pathJoin(systemTestSourceDir, "libraries", testUtilsFileName),
-			pathJoin(profileSysTestSpyPackageDir, testUtilsFileName)
+			_pJoin(systemTestSourceDir, "libraries", testUtilsFileName),
+			_pJoin(profileSysTestSpyPackageDir, testUtilsFileName)
 		)
 
 	def teardown_nvda_profile(self):
 		builtIn.log("Removing files from NVDA profile")
 		opSys.remove_file(
-			pathJoin(nvdaProfileWorkingDir, "nvda.ini")
+			_pJoin(nvdaProfileWorkingDir, "nvda.ini")
 		)
 		opSys.remove_directory(
 			profileSysTestSpyPackageDir,
@@ -92,9 +92,9 @@ def _connectToRemoteServer(self):
 		builtIn.log("Waiting for nvdaSpy to be available at: {}".format(spyServerURI))
 		# Importing the 'Remote' library always succeeds, even when a connection can not be made.
 		# If that happens, then some 'Remote' keyword will fail at some later point.
-		# therefore we use 'test_remote_server' to ensure that we can in fact connect before proceeding.
-		blockUntilConditionMet(
-			getValue=lambda: test_remote_server(spyServerURI, log=False),
+		# therefore we use '_testRemoteServer' to ensure that we can in fact connect before proceeding.
+		_blockUntilConditionMet(
+			getValue=lambda: _testRemoteServer(spyServerURI, log=False),
 			giveUpAfterSeconds=10,
 			errorMessage="Unable to connect to nvdaSpy",
 		)
@@ -140,11 +140,12 @@ def save_NVDA_log(self):
 			).replace(" ", "_")
 		opSys.copy_file(
 			nvdaLogFilePath,
-			pathJoin(outDir, "nvdaTestRunLogs", outputFileName)
+			_pJoin(outDir, "nvdaTestRunLogs", outputFileName)
 		)
 
 	def quit_NVDA(self):
 		builtIn.log("Stopping nvdaSpy server: {}".format(spyServerURI))
+		_stopRemoteServer(spyServerURI, log=False)
 		# remove the spy so that if nvda is run manually against this config it does not interfere.
 		self.teardown_nvda_profile()
 		process.run_process(
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index e1573d49d96..a97ee2ce4c9 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -10,10 +10,11 @@
 
 import globalPluginHandler
 import threading
-from testutils import blockUntilConditionMet
+from testutils import _blockUntilConditionMet
 from robotremoteserver import RobotRemoteServer
 from logHandler import log
-from timeit import default_timer as timer
+from timeit import default_timer as _timer
+
 whitespaceMinusSlashN = '\t\x0b\x0c\r '
 
 
@@ -27,7 +28,7 @@ def __init__(self):
 		self._allSpeechStartIndex = 0
 		self._speechOccurred = False
 		self.isNvdaStartupComplete = False
-		self.lastSpeechTime = timer()
+		self.lastSpeechTime = _timer()
 		self._registerWithExtensionPoints()
 
 	def _registerWithExtensionPoints(self):
@@ -44,7 +45,7 @@ def _onNvdaSpeech(self, speechSequence=None):
 		if not speechSequence: return
 		with threading.Lock():
 			self._speechOccurred = True
-			self.lastSpeechTime = timer()
+			self.lastSpeechTime = _timer()
 			self._nvdaSpeech.append(speechSequence)
 
 	# Private helper methods
@@ -94,7 +95,7 @@ def getIndexOfSpeech(self, speech, indexHint=0):
 			return -1
 
 	def hasSpeechFinished(self):
-		return self.SPEECH_HAS_FINISHED_SECONDS < timer() - self.lastSpeechTime
+		return self.SPEECH_HAS_FINISHED_SECONDS < _timer() - self.lastSpeechTime
 
 	def dumpSpeechToLog(self):
 		log.debug("All speech:\n{}".format(repr(self._nvdaSpeech)))
@@ -119,7 +120,7 @@ def set_max_keyword_duration(self, maxSeconds):
 		self._maxKeywordDuration = maxSeconds-1
 
 	def wait_for_NVDA_startup_to_complete(self):
-		blockUntilConditionMet(
+		_blockUntilConditionMet(
 			getValue=lambda: self._spy.isNvdaStartupComplete,
 			giveUpAfterSeconds=self._minTimeout(10),
 			errorMessage="Unable to connect to nvdaSpy",
@@ -146,7 +147,7 @@ def get_last_speech_index(self):
 # TODO: does this need to return the index of this speech? What if the "next speech" has already past when this is called?
 	def wait_for_speech_to_start(self, maxWaitSeconds=5.0, raiseErrorOnTimeout=True):
 		useErrorMesg = "Speech did not start before timeout" if raiseErrorOnTimeout else None
-		success, value = blockUntilConditionMet(
+		success, value = _blockUntilConditionMet(
 			getValue=lambda: self._spy.checkIfSpeechOccurredAndReset(),
 			giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
 			errorMessage=useErrorMesg
@@ -156,7 +157,7 @@ def wait_for_speech_to_start(self, maxWaitSeconds=5.0, raiseErrorOnTimeout=True)
 	def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
 		sinceIndex = 0 if not sinceIndex else sinceIndex
 		try:
-			success, speechIndex = blockUntilConditionMet(
+			success, speechIndex = _blockUntilConditionMet(
 				getValue=lambda: self._spy.getIndexOfSpeech(speech, sinceIndex),
 				giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
 				shouldStopEvaluator=lambda speechIndex: speechIndex >= 0,
@@ -169,7 +170,7 @@ def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
 		return speechIndex
 
 	def wait_for_speech_to_finish(self, maxWaitSeconds=5.0):
-		blockUntilConditionMet(
+		_blockUntilConditionMet(
 			getValue=self._spy.hasSpeechFinished,
 			giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
 			errorMessage="Speech did not finish before timeout"
diff --git a/tests/system/libraries/testutils.py b/tests/system/libraries/testutils.py
index 657a83af95d..5267594bb37 100644
--- a/tests/system/libraries/testutils.py
+++ b/tests/system/libraries/testutils.py
@@ -8,10 +8,10 @@
 package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library.
 """
 
-from timeit import default_timer as timer
+from timeit import default_timer as _timer
 
 
-def blockUntilConditionMet(
+def _blockUntilConditionMet(
 		getValue,
 		giveUpAfterSeconds,
 		shouldStopEvaluator=lambda value: bool(value),
@@ -25,13 +25,13 @@ def blockUntilConditionMet(
 	"""
 	assert callable(getValue)
 	assert callable(shouldStopEvaluator)
-	startTime = timer()
+	startTime = _timer()
 	lastRunTime = startTime
 	firstRun = True  # ensure we start immediately
-	while (timer() - startTime) < giveUpAfterSeconds:
-		if firstRun or (timer() - lastRunTime) > intervalBetweenSeconds:
+	while (_timer() - startTime) < giveUpAfterSeconds:
+		if firstRun or (_timer() - lastRunTime) > intervalBetweenSeconds:
 			firstRun = False
-			lastRunTime = timer()
+			lastRunTime = _timer()
 			val = getValue()
 			if shouldStopEvaluator(val):
 				return True, val

From 5266301610865e749703733c864a8147eeb1c15f Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Mon, 2 Jul 2018 04:37:59 +0000
Subject: [PATCH 75/95] Move non NVDA specific helpers into a new library

---
 tests/system/initial.robot             |  7 +++---
 tests/system/libraries/helperLib.py    | 34 ++++++++++++++++++++++++++
 tests/system/libraries/nvdaRobotLib.py | 19 --------------
 3 files changed, 38 insertions(+), 22 deletions(-)
 create mode 100644 tests/system/libraries/helperLib.py

diff --git a/tests/system/initial.robot b/tests/system/initial.robot
index 957f259ead5..93de5786095 100644
--- a/tests/system/initial.robot
+++ b/tests/system/initial.robot
@@ -9,6 +9,7 @@ Library	OperatingSystem
 Library	Process
 Library	libraries/sendKey.py
 Library	libraries/nvdaRobotLib.py
+Library	libraries/helperLib.py
 
 Test Setup	start NVDA	standard-dontShowWelcomeDialog.ini
 Test Teardown	quit NVDA
@@ -20,8 +21,7 @@ Ensure NVDA runs at all
 	process should be running	nvdaAlias
 
 Ensure NVDA quits from keyboard
-	# TODO: can we make a better syntax for this? Maybe: 'join two spaces	sample text'
-	${Exit NVDA dialog} =	catenate	SEPARATOR=${SPACE * 2}	Exit NVDA	dialog
+	${Exit NVDA dialog} =	catenate double space	Exit NVDA	dialog
 	send key	insert	q
 	${INDEX} =	wait for specific speech	${Exit NVDA dialog}
 
@@ -34,7 +34,8 @@ Ensure NVDA quits from keyboard
 
 Can read the welcome dialog
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
-	${Welcome dialog title} =	catenate	SEPARATOR=${SPACE * 2}	Welcome to NVDA	dialog
+
+	${Welcome dialog title} =	catenate double space	Welcome to NVDA	dialog
 	${INDEX} =	wait for specific speech	${Welcome dialog title}
 	wait for speech to finish
 	${actual speech} =	get speech from index until now	${INDEX}
diff --git a/tests/system/libraries/helperLib.py b/tests/system/libraries/helperLib.py
new file mode 100644
index 00000000000..bd791983b03
--- /dev/null
+++ b/tests/system/libraries/helperLib.py
@@ -0,0 +1,34 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
+
+"""This file provides general robot library functions for system tests.
+This is in contrast with nvdaRobotLib.py which contains helpers related to starting and stopping NVDA for system
+tests, or with systemTestSpy which contains methods for extracting information about NVDA's behaviour during system
+tests.
+"""
+from robot.libraries.BuiltIn import BuiltIn
+builtIn = BuiltIn()  # type: BuiltIn
+
+def assert_strings_are_equal( actual, expected, ignore_case=False):
+	try:
+		builtIn.should_be_equal_as_strings(
+			actual,
+			expected,
+			msg="Actual speech != Expected speech",
+			ignore_case=ignore_case
+		)
+	except AssertionError:
+		builtIn.log(
+			"repr of actual vs expected (ignore_case={}):\n{}\nvs\n{}".format(
+				ignore_case,
+				repr(actual),
+				repr(expected)
+			)
+		)
+		raise
+
+
+def catenate_double_space(*args):
+	return "  ".join(args)
\ No newline at end of file
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index f7e3fe7a817..1261fbe969c 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -155,22 +155,3 @@ def quit_NVDA(self):
 		)
 		process.wait_for_process(self.nvdaHandle)
 		self.save_NVDA_log()
-
-# TODO: this shouldn't be a member function, but it is not available from robot if it is not??
-	def assert_strings_are_equal(self, actual, expected, ignore_case=False):
-		try:
-			builtIn.should_be_equal_as_strings(
-				actual,
-				expected,
-				msg="Actual speech != Expected speech",
-				ignore_case=ignore_case
-			)
-		except AssertionError:
-			builtIn.log(
-				"repr of actual vs expected (ignore_case={}):\n{}\nvs\n{}".format(
-					ignore_case,
-					repr(actual),
-					repr(expected)
-				)
-			)
-			raise
\ No newline at end of file

From fd6738333db6f50061809b15c71f6a5f317a5c37 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Tue, 3 Jul 2018 11:01:39 +0000
Subject: [PATCH 76/95] Create self contained package for system test spy

The system test spy now copies in library dependencies that are not
provided by a NVDA install. This is so that the system tests can be run
on appVeyor against an installed version of NVDA
---
 source/setup.py                               |  2 +-
 tests/system/libraries/nvdaRobotLib.py        | 81 ++++++++++++++-----
 tests/system/libraries/systemTestSpy.py       | 16 +++-
 .../{testutils.py => systemtestutils.py}      |  2 +-
 4 files changed, 77 insertions(+), 24 deletions(-)
 rename tests/system/libraries/{testutils.py => systemtestutils.py} (94%)

diff --git a/source/setup.py b/source/setup.py
index 761cbc33b25..92726fdfa0a 100755
--- a/source/setup.py
+++ b/source/setup.py
@@ -207,7 +207,7 @@ def getRecursiveDataFiles(dest,source,excludes=()):
 		# #3368: bisect was implicitly included with Python 2.7.3, but isn't with 2.7.5.
 		# Also, the service executable used win32api, which some add-ons use for various purposes.
 		# Explicitly include them so we don't break some add-ons.
-		"includes": ["nvdaBuiltin", "bisect", "win32api","robotremoteserver"],
+		"includes": ["nvdaBuiltin", "bisect", "win32api"],
 	}},
 	data_files=[
 		(".",glob("*.dll")+glob("*.manifest")+["builtin.dic"]),
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index f377f54ac0c..6827223ea45 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -8,10 +8,10 @@
 the systemTestSpy.py file, which provides library functions related to monitoring / asserting NVDA output.
 """
 # imported methods start with underscore (_) so they don't get imported into robot files as keywords
-from os.path import join as _pJoin
-from os.path import abspath as _abspath
+from os.path import join as _pJoin, abspath as _abspath, expandvars as _expandvars
+import sys
 from robotremoteserver import test_remote_server as _testRemoteServer, stop_remote_server as _stopRemoteServer
-from testutils import _blockUntilConditionMet
+from systemTestUtils import _blockUntilConditionMet
 from robot.libraries.BuiltIn import BuiltIn
 from robot.libraries.OperatingSystem import OperatingSystem
 from robot.libraries.Process import Process
@@ -24,18 +24,65 @@
 spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
 spyAlias = "nvdaSpy"
 
-# Paths
+
 whichNVDA=builtIn.get_variable_value("${whichNVDA}", "source")
 if whichNVDA=="source":
 	baseNVDACommandline="pythonw source/nvda.pyw"
 elif whichNVDA=="installed":
-	baseNVDACommandline='"%s"'%_pJoin(os.path.expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
+	baseNVDACommandline='"%s"'%_pJoin(_expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
 print baseNVDACommandline
+
+# Paths
 systemTestSourceDir = _abspath("tests/system")
 nvdaProfileWorkingDir = _pJoin(systemTestSourceDir, "nvdaProfile")
 nvdaLogFilePath = _pJoin(nvdaProfileWorkingDir,'nvda.log')
-profileGlobalPluginsDir = _pJoin(nvdaProfileWorkingDir, "globalPlugins")
-profileSysTestSpyPackageDir = _pJoin(profileGlobalPluginsDir, "systemTestSpy")
+systemTestSpyAddonName = "systemTestSpy"
+testSpyPackageDest = _pJoin(nvdaProfileWorkingDir, "globalPlugins")
+
+
+def _findDepPath(depFileName, searchPaths):
+	import os
+	print searchPaths
+	for path in searchPaths:
+		filePath = _pJoin(path, depFileName+".py")
+		print filePath
+		if os.path.isfile(filePath):
+			return filePath
+		elif os.path.isfile(_pJoin(path, depFileName, "__init__.py")):
+			return _pJoin(path, depFileName)
+	raise AssertionError("Unable to find required system test spy dependency: {}".format(depFileName))
+
+# relative to the python path
+requiredPythonImports = [
+	r"robotremoteserver",
+	r"SimpleXMLRPCServer",
+	r"xmlrpclib",
+]
+
+def _createNvdaSpyPackage():
+	import os
+	searchPaths = sys.path
+	profileSysTestSpyPackageStagingDir = _pJoin(systemTestSourceDir, systemTestSpyAddonName)
+	# copy in required dependencies, the addon will modify the python path
+	# to point to this sub dir
+	spyPackageLibsDir = _pJoin(profileSysTestSpyPackageStagingDir, "libs")
+	opSys.create_directory(spyPackageLibsDir)
+	for lib in requiredPythonImports:
+		libSource = _findDepPath(lib, searchPaths)
+		if os.path.isdir(libSource):
+			opSys.copy_directory(libSource, spyPackageLibsDir)
+		elif os.path.isfile(libSource):
+			opSys.copy_file(libSource, spyPackageLibsDir)
+
+	opSys.copy_file(
+		_pJoin(systemTestSourceDir, "libraries", systemTestSpyAddonName+".py"),
+		_pJoin(profileSysTestSpyPackageStagingDir, "__init__.py")
+	)
+	opSys.copy_file(
+		_pJoin(systemTestSourceDir, "libraries", "systemTestUtils.py"),
+		profileSysTestSpyPackageStagingDir
+	)
+	return profileSysTestSpyPackageStagingDir
 
 
 class nvdaRobotLib(object):
@@ -51,15 +98,9 @@ def setup_nvda_profile(self, settingsFileName):
 			_pJoin(nvdaProfileWorkingDir, "nvda.ini")
 		)
 		# create a package to use as the globalPlugin
-		opSys.create_directory(profileSysTestSpyPackageDir)
-		opSys.copy_file(
-			_pJoin(systemTestSourceDir, "libraries", "systemTestSpy.py"),
-			_pJoin(profileSysTestSpyPackageDir, "__init__.py")
-		)
-		testUtilsFileName = "testutils.py"
-		opSys.copy_file(
-			_pJoin(systemTestSourceDir, "libraries", testUtilsFileName),
-			_pJoin(profileSysTestSpyPackageDir, testUtilsFileName)
+		opSys.copy_directory(
+			_createNvdaSpyPackage(),
+			_pJoin(testSpyPackageDest, systemTestSpyAddonName)
 		)
 
 	def teardown_nvda_profile(self):
@@ -68,7 +109,7 @@ def teardown_nvda_profile(self):
 			_pJoin(nvdaProfileWorkingDir, "nvda.ini")
 		)
 		opSys.remove_directory(
-			profileSysTestSpyPackageDir,
+			testSpyPackageDest,
 			recursive=True
 		)
 
@@ -77,7 +118,11 @@ def _startNVDAProcess(self):
 		Use debug logging, replacing any current instance, using the system test profile directory
 		"""
 		self.nvdaHandle = handle = process.start_process(
-			"{baseNVDACommandline} --debug-logging -r -c \"{nvdaProfileDir}\" --log-file \"{nvdaLogFilePath}\"".format(baseNVDACommandline=baseNVDACommandline,nvdaProfileDir=nvdaProfileWorkingDir,nvdaLogFilePath=nvdaLogFilePath),
+			"{baseNVDACommandline} --debug-logging -r -c \"{nvdaProfileDir}\" --log-file \"{nvdaLogFilePath}\"".format(
+				baseNVDACommandline=baseNVDACommandline,
+				nvdaProfileDir=nvdaProfileWorkingDir,
+				nvdaLogFilePath=nvdaLogFilePath
+			),
 			shell=True,
 			alias='nvdaAlias'
 		)
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index a97ee2ce4c9..97803617984 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -7,13 +7,21 @@
 It is copied into the (system test specific) NVDA profile directory. It becomes the '__init__.py' file as part of a
 package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library.
 """
-
 import globalPluginHandler
 import threading
-from testutils import _blockUntilConditionMet
-from robotremoteserver import RobotRemoteServer
+from systemTestUtils import _blockUntilConditionMet
 from logHandler import log
-from timeit import default_timer as _timer
+from time import clock as _timer
+
+import sys
+import os
+log.debug("before pathmod: {}".format(sys.path))
+# Get the path to the top of the package
+TOP_DIR = os.path.abspath(os.path.dirname(__file__))
+# imports that require libraries not distributed with an install of NVDA
+sys.path.append( os.path.join(TOP_DIR, "libs"))
+log.debug("after pathmod: {}".format(sys.path))
+from robotremoteserver import RobotRemoteServer
 
 whitespaceMinusSlashN = '\t\x0b\x0c\r '
 
diff --git a/tests/system/libraries/testutils.py b/tests/system/libraries/systemtestutils.py
similarity index 94%
rename from tests/system/libraries/testutils.py
rename to tests/system/libraries/systemtestutils.py
index 5267594bb37..bb88fb16d8f 100644
--- a/tests/system/libraries/testutils.py
+++ b/tests/system/libraries/systemtestutils.py
@@ -8,7 +8,7 @@
 package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library.
 """
 
-from timeit import default_timer as _timer
+from time import clock as _timer
 
 
 def _blockUntilConditionMet(

From f1be08f50524f3ebbaeecc8822837566fc5a6988 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Tue, 3 Jul 2018 12:31:01 +0000
Subject: [PATCH 77/95] Zip system test output before pushing to artifacts

---
 appveyor.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 12f08115334..f970ed482ab 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -150,10 +150,8 @@ test_script:
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
  - ps: |
      py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml -v whichNVDA:installed tests/system/
-     Get-ChildItem ".\output/testResults/system\" -Attributes !Directory -depth 2 |
-      Foreach-Object {
-          Push-AppveyorArtifact $_.FullName
-      }
+     Compress-Archive -Path ".\output\testResults\system\*" -DestinationPath ".\output\testResults\systemTestResult.zip"
+     Push-AppveyorArtifact ".\output\testResults\systemTestResult.zip"
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\system\systemTests.xml))

From da14b4beac445b7f67c5e35857e7d8bc021f017b Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Tue, 3 Jul 2018 12:51:47 +0000
Subject: [PATCH 78/95] try adding robotlibraries directory to the path

---
 tests/system/libraries/nvdaRobotLib.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 6827223ea45..fa7b36f6e6d 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -11,10 +11,15 @@
 from os.path import join as _pJoin, abspath as _abspath, expandvars as _expandvars
 import sys
 from robotremoteserver import test_remote_server as _testRemoteServer, stop_remote_server as _stopRemoteServer
-from systemTestUtils import _blockUntilConditionMet
 from robot.libraries.BuiltIn import BuiltIn
 from robot.libraries.OperatingSystem import OperatingSystem
 from robot.libraries.Process import Process
+import os
+sys.path.append(
+	os.path.abspath(
+		os.path.dirname(__file__))
+)
+from systemTestUtils import _blockUntilConditionMet
 
 builtIn = BuiltIn()  # type: BuiltIn
 process = builtIn.get_library_instance('Process')  # type: Process

From 6818faa8362f604daa00f670a84190a6dc7751a9 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Tue, 3 Jul 2018 23:17:45 +0000
Subject: [PATCH 79/95] Fix case of file name

This should resolve the appveyor system test failures
---
 tests/system/libraries/nvdaRobotLib.py                       | 5 -----
 .../libraries/{systemtestutils.py => systemTestUtils.py}     | 0
 2 files changed, 5 deletions(-)
 rename tests/system/libraries/{systemtestutils.py => systemTestUtils.py} (100%)

diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index fa7b36f6e6d..9d33fc76ad4 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -14,11 +14,6 @@
 from robot.libraries.BuiltIn import BuiltIn
 from robot.libraries.OperatingSystem import OperatingSystem
 from robot.libraries.Process import Process
-import os
-sys.path.append(
-	os.path.abspath(
-		os.path.dirname(__file__))
-)
 from systemTestUtils import _blockUntilConditionMet
 
 builtIn = BuiltIn()  # type: BuiltIn
diff --git a/tests/system/libraries/systemtestutils.py b/tests/system/libraries/systemTestUtils.py
similarity index 100%
rename from tests/system/libraries/systemtestutils.py
rename to tests/system/libraries/systemTestUtils.py

From 39f6b07a080e977adfa820fa3c1494bbd70c5180 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 4 Jul 2018 12:27:22 +0000
Subject: [PATCH 80/95] further tidy up

---
 appveyor.yml                              | 14 +++----
 tests/system/libraries/nvdaRobotLib.py    | 17 ++++----
 tests/system/libraries/systemTestSpy.py   | 19 ++-------
 tests/system/libraries/systemTestUtils.py |  5 +++
 tests/system/readme.md                    | 51 ++++++++++++++++++++---
 5 files changed, 69 insertions(+), 37 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index f970ed482ab..06d7c01fa9d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -142,19 +142,19 @@ before_test:
 test_script:
  - ps: |
      $errorCode=0
-     py -m nose --with-xunit --xunit-file=output/testResults/unitTests.xml tests/unit
+     py -m nose --with-xunit --xunit-file=testOutput/unit/unitTests.xml tests/unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
-     Push-AppveyorArtifact .\output\testResults\unitTests.xml
+     Push-AppveyorArtifact .\testOutput\unit\unitTests.xml
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\unitTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testOutput\unit\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
  - ps: |
-     py -m robot --loglevel DEBUG -d output/testResults/system -x systemTests.xml -v whichNVDA:installed tests/system/
-     Compress-Archive -Path ".\output\testResults\system\*" -DestinationPath ".\output\testResults\systemTestResult.zip"
-     Push-AppveyorArtifact ".\output\testResults\systemTestResult.zip"
+     py -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:installed tests/system/
+     Compress-Archive -Path ".\testOutput\system\*" -DestinationPath ".\testOutput\systemTestResult.zip"
+     Push-AppveyorArtifact ".\testOutput\systemTestResult.zip"
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\output\testResults\system\systemTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testOutput\system\systemTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
 
 artifacts:
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 9d33fc76ad4..ad79380ab47 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -30,12 +30,13 @@
 	baseNVDACommandline="pythonw source/nvda.pyw"
 elif whichNVDA=="installed":
 	baseNVDACommandline='"%s"'%_pJoin(_expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
-print baseNVDACommandline
+else:
+	raise AssertionError("robot should be given argument `-v whichNVDA [source|installed]")
 
 # Paths
 systemTestSourceDir = _abspath("tests/system")
-nvdaProfileWorkingDir = _pJoin(systemTestSourceDir, "nvdaProfile")
-nvdaLogFilePath = _pJoin(nvdaProfileWorkingDir,'nvda.log')
+nvdaProfileWorkingDir = _pJoin(_expandvars('%TEMP%'), "nvdaProfile")
+nvdaLogFilePath = _pJoin(nvdaProfileWorkingDir, 'nvda.log')
 systemTestSpyAddonName = "systemTestSpy"
 testSpyPackageDest = _pJoin(nvdaProfileWorkingDir, "globalPlugins")
 
@@ -53,7 +54,7 @@ def _findDepPath(depFileName, searchPaths):
 	raise AssertionError("Unable to find required system test spy dependency: {}".format(depFileName))
 
 # relative to the python path
-requiredPythonImports = [
+requiredPythonImportsForSystemTestSpyPackage = [
 	r"robotremoteserver",
 	r"SimpleXMLRPCServer",
 	r"xmlrpclib",
@@ -62,12 +63,12 @@ def _findDepPath(depFileName, searchPaths):
 def _createNvdaSpyPackage():
 	import os
 	searchPaths = sys.path
-	profileSysTestSpyPackageStagingDir = _pJoin(systemTestSourceDir, systemTestSpyAddonName)
+	profileSysTestSpyPackageStagingDir = _pJoin(_expandvars('%TEMP%'), systemTestSpyAddonName)
 	# copy in required dependencies, the addon will modify the python path
 	# to point to this sub dir
 	spyPackageLibsDir = _pJoin(profileSysTestSpyPackageStagingDir, "libs")
 	opSys.create_directory(spyPackageLibsDir)
-	for lib in requiredPythonImports:
+	for lib in requiredPythonImportsForSystemTestSpyPackage:
 		libSource = _findDepPath(lib, searchPaths)
 		if os.path.isdir(libSource):
 			opSys.copy_directory(libSource, spyPackageLibsDir)
@@ -196,11 +197,11 @@ def save_NVDA_log(self):
 	def quit_NVDA(self):
 		builtIn.log("Stopping nvdaSpy server: {}".format(spyServerURI))
 		_stopRemoteServer(spyServerURI, log=False)
-		# remove the spy so that if nvda is run manually against this config it does not interfere.
-		self.teardown_nvda_profile()
 		process.run_process(
 			"{baseNVDACommandline} -q --disable-addons".format(baseNVDACommandline=baseNVDACommandline),
 			shell=True,
 		)
 		process.wait_for_process(self.nvdaHandle)
 		self.save_NVDA_log()
+		# remove the spy so that if nvda is run manually against this config it does not interfere.
+		self.teardown_nvda_profile()
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index 97803617984..1021b32eda4 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -90,16 +90,13 @@ def getIndexOfLastSpeech(self):
 		with threading.Lock():
 			return len(self._nvdaSpeech) - 1
 
-	def getIndexOfSpeech(self, speech, indexHint=0):
+	def getIndexOfSpeech(self, speech, startFromIndex=0):
 		with threading.Lock():
-			log.debug("from index:{}, looking for speech: {}".format(indexHint, speech))
-			for index, commands in enumerate(self._nvdaSpeech[indexHint:]):
-				index = index+indexHint
+			for index, commands in enumerate(self._nvdaSpeech[startFromIndex:]):
+				index = index + startFromIndex
 				baseStrings = [c.strip() for c in commands if isinstance(c, basestring)]
 				if any(speech in x for x in baseStrings):
-					log.debug("found it")
 					return index
-			log.debug("did not find it")
 			return -1
 
 	def hasSpeechFinished(self):
@@ -152,16 +149,6 @@ def reset_all_speech_index(self):
 	def get_last_speech_index(self):
 		return self._spy.getIndexOfLastSpeech()
 
-# TODO: does this need to return the index of this speech? What if the "next speech" has already past when this is called?
-	def wait_for_speech_to_start(self, maxWaitSeconds=5.0, raiseErrorOnTimeout=True):
-		useErrorMesg = "Speech did not start before timeout" if raiseErrorOnTimeout else None
-		success, value = _blockUntilConditionMet(
-			getValue=lambda: self._spy.checkIfSpeechOccurredAndReset(),
-			giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
-			errorMessage=useErrorMesg
-		)
-		return success
-
 	def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
 		sinceIndex = 0 if not sinceIndex else sinceIndex
 		try:
diff --git a/tests/system/libraries/systemTestUtils.py b/tests/system/libraries/systemTestUtils.py
index bb88fb16d8f..709c47acdc2 100644
--- a/tests/system/libraries/systemTestUtils.py
+++ b/tests/system/libraries/systemTestUtils.py
@@ -8,6 +8,7 @@
 package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library.
 """
 
+from time import sleep as _sleep
 from time import clock as _timer
 
 
@@ -25,6 +26,8 @@ def _blockUntilConditionMet(
 	"""
 	assert callable(getValue)
 	assert callable(shouldStopEvaluator)
+	assert intervalBetweenSeconds > 0.001
+	SLEEP_TIME = intervalBetweenSeconds * 0.5
 	startTime = _timer()
 	lastRunTime = startTime
 	firstRun = True  # ensure we start immediately
@@ -35,6 +38,8 @@ def _blockUntilConditionMet(
 			val = getValue()
 			if shouldStopEvaluator(val):
 				return True, val
+			_sleep(SLEEP_TIME)
+
 	else:
 		if errorMessage:
 			raise AssertionError(errorMessage)
diff --git a/tests/system/readme.md b/tests/system/readme.md
index 7112bda3653..bafb40caec0 100644
--- a/tests/system/readme.md
+++ b/tests/system/readme.md
@@ -18,20 +18,59 @@ pip install pyautogui
 
 ### Running the tests
 
-These tests should be run from the windows command prompt (cmd.exe).
+These tests should be run from the windows command prompt (cmd.exe) from the root directory of your NVDA repository.
 
-From the root directory of your NVDA repository, run:
+The `whichNVDA` argument allows the tests to be run against an installed copy
+of NVDA (first ensure it is compatible with the tests). Note valid values are:
+* "installed" - when running against the installed version of NVDA, you are likely to get errors in the log unless
+the tests are run from an administrator command prompt.
+* "source"
 
 ```
-python -m robot tests/system/
+python -m robot --loglevel DEBUG -d testOutput -v whichNVDA:source tests/system/
 ```
 
-To run a single test:
+To run a single test, use the `--test` argument. Refer to the robot framework documentation for further details.
 
 ```
-python -m robot --test "name of test here" tests/system/
+python -m robot --loglevel DEBUG -d testOutput -v whichNVDA:source --test "name of test here" tests/system/
 ```
 
 ### Getting the results
 
-The process is displayed in the command prompt, for more information consider the `report.html`, `log.html`, and `output.xml` files.
\ No newline at end of file
+The process is displayed in the command prompt, for more information consider the
+`report.html`, `log.html`, and `output.xml` files. The logs from NVDA are saved to the `nvdaTestRunLogs` folder
+
+### Overview
+
+Robot Framework loads and parses the test files and their libraries. In our case, generally in the 'setup',
+NVDA is started as a new process. It uses a sand box profile, and communication with the test code occurs via an
+NVDA addon (`systemTestSpy`). The system test should, as much as possible, interact like a user would. For example,
+wait for the speech to confirm that an expected dialog is open before taking the next action to interact.
+
+Test code goes in robot files, see the robot framework documentation to understand these.
+Large strings or other variables go in a variables.py file paired with the robot file.
+The `libraries` directory contains files providing "robot keyword" libraries.
+The `nvdaSettingsFiles` directory contains various NVDA config files that are used to construct the NVDA
+profile in the `%TEMP%` directory.
+
+### How the test setup works
+
+This section will not go into the details of robot framework, or robot remote server,
+these have their own documentation. There are two major libraries used with the system tests:
+
+* `nvdaRobotLib` - Provides keywords which help the test code start and cleanup the NVDA process, including the installation of the `systemTestSpy` addon.
+* `systemTestSpy` - Is converted into an addon that is installed in the NVDA profile used with the version of NVDA under test. This provides keywords for getting information out of NVDA. For example getting the last speech.
+
+Helper code in `nvdaRobotLib` is responsible for the construction of the `systemTestSpy` addon.
+The addon is constructed as a package from several files:
+* `libraries/systemTestSpy.py` becomes `systemTestSpy/__init__.py`
+* `libraries/systemTestUtils.py` becomes `systemTestSpy/systemTestUtils.py`
+* Files listed in `nvdaRobotLib.requiredPythonImportsForSystemTestSpyPackage` are sourced from locations listed
+in the python paths for the instance of python running the robot tests. These are copied to `systemTestSpy/libs/`
+
+An NVDA profile directory is created in the `%TEMP%` directory, the `systemTestSpy` addon is copied
+into the `globalPlugins` directory of this NVDA profile. For each test, an NVDA configuration file
+is copied into this profile as well. NVDA is started, and points to this profile directory. At the end of the
+test the NVDA log is copied to the robot output directory, under the `nvdaTestRunLogs` directory. The log files are
+named by suite and test name.
\ No newline at end of file

From 50137bc4ac4976418cc533e7495c7f16f7b9cd4c Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 4 Jul 2018 12:47:37 +0000
Subject: [PATCH 81/95] Introduce suite name

---
 .../startupShutdownNVDA.robot}                 | 18 ++++++++++--------
 tests/system/{ => NVDA Core}/variables.py      |  0
 2 files changed, 10 insertions(+), 8 deletions(-)
 rename tests/system/{initial.robot => NVDA Core/startupShutdownNVDA.robot} (84%)
 rename tests/system/{ => NVDA Core}/variables.py (100%)

diff --git a/tests/system/initial.robot b/tests/system/NVDA Core/startupShutdownNVDA.robot
similarity index 84%
rename from tests/system/initial.robot
rename to tests/system/NVDA Core/startupShutdownNVDA.robot
index b9a701e76dc..fcdeb0073b4 100644
--- a/tests/system/initial.robot
+++ b/tests/system/NVDA Core/startupShutdownNVDA.robot	
@@ -7,9 +7,9 @@ Documentation	Basic start and exit tests
 
 Library	OperatingSystem
 Library	Process
-Library	libraries/sendKey.py
-Library	libraries/nvdaRobotLib.py
-Library	libraries/helperLib.py
+Library	../libraries/sendKey.py
+Library	../libraries/nvdaRobotLib.py
+Library	../libraries/helperLib.py
 
 Test Setup	start NVDA	standard-dontShowWelcomeDialog.ini
 Test Teardown	quit NVDA
@@ -17,16 +17,18 @@ Test Teardown	quit NVDA
 Variables	variables.py
 
 *** Test Cases ***
-Ensure NVDA runs at all
+Starts
 	process should be running	nvdaAlias
 
-Ensure NVDA quits from keyboard
+Quits from keyboard
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
-	
+
 	${Welcome dialog title} =	catenate double space	Welcome to NVDA	dialog
 	wait for specific speech	${Welcome dialog title}
+	wait for speech to finish
+	sleep	1	# the dialog is not always receiving the enter keypress, wait a little longer for it
 	send key	enter
-	
+
 	${Exit NVDA dialog} =	catenate double space	Exit NVDA	dialog
 	send key	insert	q
 	${INDEX} =	wait for specific speech	${Exit NVDA dialog}
@@ -39,7 +41,7 @@ Ensure NVDA quits from keyboard
 	wait for process	nvdaAlias	timeout=10 sec
 	process should be stopped	nvdaAlias
 
-Can read the welcome dialog
+Read welcome dialog
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
 
 	${Welcome dialog title} =	catenate double space	Welcome to NVDA	dialog
diff --git a/tests/system/variables.py b/tests/system/NVDA Core/variables.py
similarity index 100%
rename from tests/system/variables.py
rename to tests/system/NVDA Core/variables.py

From cbf6c7e0fa05f24f2533df0129173409d3a7ccd4 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 4 Jul 2018 12:48:24 +0000
Subject: [PATCH 82/95] fix appveyor system tests

testOutput directories need to be created before writting files to them!
---
 appveyor.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 06d7c01fa9d..8cc27892420 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -116,8 +116,9 @@ before_test:
  - py -m pip install robotremoteserver
  - py -m pip install pyautogui
  - py -m pip install nose 
- - mkdir output\testResults
- - mkdir output\testResults\system
+ - mkdir testOutput
+ - mkdir testOutput\unit
+ - mkdir testOutput\system
  - ps: |
      $errorCode=0
      $nvdaLauncherFile=".\output\nvda"

From 385e8ead39389852f74462407acf85b7cd99bbc4 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 4 Jul 2018 13:04:03 +0000
Subject: [PATCH 83/95] Stop double nvda install log artifact

---
 appveyor.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 8cc27892420..ac2f1d0b70d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -127,8 +127,8 @@ before_test:
      }
      $nvdaLauncherFile+="_${env:version}.exe"
      echo NVDALauncherFile: $NVDALauncherFile
-     $outputDir=$(resolve-path .\output)
-          $installerLogFilePath="$outputDir\nvda_install.log"
+     $outputDir=$(resolve-path .\testOutput)
+     $installerLogFilePath="$outputDir\nvda_install.log"
      $installerProcess=start-process -FilePath "$nvdaLauncherFile" -ArgumentList "--install-silent --debug-logging --log-file $installerLogFilePath" -passthru
      try {
       $installerProcess | wait-process -Timeout 180
@@ -137,7 +137,7 @@ before_test:
       echo NVDA installer process timed out
       $errorCode=1
      }
-          Push-AppveyorArtifact $installerLogFilePath
+     Push-AppveyorArtifact $installerLogFilePath
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
 
 test_script:

From df023776ec98054cee6621d1b9ecf0793922aea0 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 4 Jul 2018 13:26:59 +0000
Subject: [PATCH 84/95] set robot library path

---
 appveyor.yml                                     | 3 +--
 tests/system/NVDA Core/startupShutdownNVDA.robot | 7 ++++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index ac2f1d0b70d..d0ff1cc512d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -116,7 +116,6 @@ before_test:
  - py -m pip install robotremoteserver
  - py -m pip install pyautogui
  - py -m pip install nose 
- - mkdir testOutput
  - mkdir testOutput\unit
  - mkdir testOutput\system
  - ps: |
@@ -150,7 +149,7 @@ test_script:
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testOutput\unit\unitTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
  - ps: |
-     py -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:installed tests/system/
+     py -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:installed -P tests/system/libraries tests/system/
      Compress-Archive -Path ".\testOutput\system\*" -DestinationPath ".\testOutput\systemTestResult.zip"
      Push-AppveyorArtifact ".\testOutput\systemTestResult.zip"
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
diff --git a/tests/system/NVDA Core/startupShutdownNVDA.robot b/tests/system/NVDA Core/startupShutdownNVDA.robot
index fcdeb0073b4..fdc1d594e64 100644
--- a/tests/system/NVDA Core/startupShutdownNVDA.robot	
+++ b/tests/system/NVDA Core/startupShutdownNVDA.robot	
@@ -4,12 +4,13 @@
 # For more details see: https://www.gnu.org/licenses/gpl-2.0.html
 *** Settings ***
 Documentation	Basic start and exit tests
+Default Tags	NVDA	smoke test
 
 Library	OperatingSystem
 Library	Process
-Library	../libraries/sendKey.py
-Library	../libraries/nvdaRobotLib.py
-Library	../libraries/helperLib.py
+Library	sendKey.py
+Library	nvdaRobotLib.py
+Library	helperLib.py
 
 Test Setup	start NVDA	standard-dontShowWelcomeDialog.ini
 Test Teardown	quit NVDA

From 44d94fd73f8c92ca4710f9d8e04a23b0b978fc7a Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Wed, 4 Jul 2018 13:37:16 +0000
Subject: [PATCH 85/95] reduce suite verbosity on appveyor

---
 appveyor.yml           | 21 +++++++++++++--------
 tests/system/readme.md |  4 ++--
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index d0ff1cc512d..e11f55f4ed9 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -115,7 +115,7 @@ before_test:
  - py -m pip install robotframework
  - py -m pip install robotremoteserver
  - py -m pip install pyautogui
- - py -m pip install nose 
+ - py -m pip install nose
  - mkdir testOutput\unit
  - mkdir testOutput\system
  - ps: |
@@ -142,19 +142,24 @@ before_test:
 test_script:
  - ps: |
      $errorCode=0
-     py -m nose --with-xunit --xunit-file=testOutput/unit/unitTests.xml tests/unit
+     $unitTestsXml = (Resolve-Path .\testOutput\unit\unitTests.xml)
+     cd tests
+     py -m nose --with-xunit --xunit-file=testOutput/unit/unitTests.xml ./unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
-     Push-AppveyorArtifact .\testOutput\unit\unitTests.xml
+     Push-AppveyorArtifact $unitTestsXml
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testOutput\unit\unitTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", $unitTestsXml)
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
  - ps: |
-     py -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:installed -P tests/system/libraries tests/system/
-     Compress-Archive -Path ".\testOutput\system\*" -DestinationPath ".\testOutput\systemTestResult.zip"
-     Push-AppveyorArtifact ".\testOutput\systemTestResult.zip"
+     $testOutput = (Resolve-Path .\testOutput\)
+     $systemTestOutput = (Resolve-Path .\testOutput\system)
+     cd tests
+     py -m robot --loglevel DEBUG -d $systemTestOutput -x systemTests.xml -v whichNVDA:installed -P ./system/libraries ./system/
+     Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip"
+     Push-AppveyorArtifact "$testOutput\systemTestResult.zip"
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\testOutput\system\systemTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path $systemTestOutput\systemTests.xml))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
 
 artifacts:
diff --git a/tests/system/readme.md b/tests/system/readme.md
index bafb40caec0..4d60af25045 100644
--- a/tests/system/readme.md
+++ b/tests/system/readme.md
@@ -27,13 +27,13 @@ the tests are run from an administrator command prompt.
 * "source"
 
 ```
-python -m robot --loglevel DEBUG -d testOutput -v whichNVDA:source tests/system/
+python -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:source -P tests/system/libraries tests/system/
 ```
 
 To run a single test, use the `--test` argument. Refer to the robot framework documentation for further details.
 
 ```
-python -m robot --loglevel DEBUG -d testOutput -v whichNVDA:source --test "name of test here" tests/system/
+python -m robot --test "name of test here" ...
 ```
 
 ### Getting the results

From 25f18979960dba1dd1f86b3cf4b34fcacc906b38 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Thu, 5 Jul 2018 06:56:12 +0000
Subject: [PATCH 86/95] Revert to running from repo root

Running from tests/system/ did not reduce the number of suites mentioned in Robots reports
---
 appveyor.yml                           | 11 ++++++-----
 tests/system/libraries/nvdaRobotLib.py | 21 +++++++++++++--------
 tests/system/readme.md                 | 12 +++++++-----
 3 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index e11f55f4ed9..73950120257 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -116,6 +116,7 @@ before_test:
  - py -m pip install robotremoteserver
  - py -m pip install pyautogui
  - py -m pip install nose
+ - mkdir testOutput
  - mkdir testOutput\unit
  - mkdir testOutput\system
  - ps: |
@@ -144,7 +145,7 @@ test_script:
      $errorCode=0
      $unitTestsXml = (Resolve-Path .\testOutput\unit\unitTests.xml)
      cd tests
-     py -m nose --with-xunit --xunit-file=testOutput/unit/unitTests.xml ./unit
+     py -m nose --with-xunit --xunit-file=$unitTestsXml ./unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      Push-AppveyorArtifact $unitTestsXml
      $wc = New-Object 'System.Net.WebClient'
@@ -152,14 +153,14 @@ test_script:
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
  - ps: |
      $testOutput = (Resolve-Path .\testOutput\)
-     $systemTestOutput = (Resolve-Path .\testOutput\system)
-     cd tests
-     py -m robot --loglevel DEBUG -d $systemTestOutput -x systemTests.xml -v whichNVDA:installed -P ./system/libraries ./system/
+     $systemTestOutput = (Resolve-Path "$testOutput\system")
+     $testSource = "./tests/system"
+     py -m robot --loglevel DEBUG -d $systemTestOutput -x systemTests.xml -v whichNVDA:installed -P "$testSource/libraries" "$testSource"
      Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip"
      Push-AppveyorArtifact "$testOutput\systemTestResult.zip"
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      $wc = New-Object 'System.Net.WebClient'
-     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path $systemTestOutput\systemTests.xml))
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path "$systemTestOutput\systemTests.xml"))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
 
 artifacts:
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index ad79380ab47..faff895d051 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -24,17 +24,21 @@
 spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
 spyAlias = "nvdaSpy"
 
-
-whichNVDA=builtIn.get_variable_value("${whichNVDA}", "source")
-if whichNVDA=="source":
-	baseNVDACommandline="pythonw source/nvda.pyw"
-elif whichNVDA=="installed":
-	baseNVDACommandline='"%s"'%_pJoin(_expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
+# robot is expected to be run from the NVDA repo root directory. We want all repo specific
+# paths to be relative to this. This would allow us to change where it is run from if we decided to.
+repoRoot = _abspath("./")
+whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
+if whichNVDA == "source":
+	NVDACommandPathToCheckExists = _pJoin(repoRoot, "source/nvda.pyw")
+	baseNVDACommandline = "pythonw "+NVDACommandPathToCheckExists
+elif whichNVDA == "installed":
+	NVDACommandPathToCheckExists = _pJoin(_expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
+	baseNVDACommandline='"%s"' % NVDACommandPathToCheckExists
 else:
 	raise AssertionError("robot should be given argument `-v whichNVDA [source|installed]")
 
 # Paths
-systemTestSourceDir = _abspath("tests/system")
+systemTestSourceDir = _pJoin(repoRoot, "tests", "system")
 nvdaProfileWorkingDir = _pJoin(_expandvars('%TEMP%'), "nvdaProfile")
 nvdaLogFilePath = _pJoin(nvdaProfileWorkingDir, 'nvda.log')
 systemTestSpyAddonName = "systemTestSpy"
@@ -99,7 +103,7 @@ def setup_nvda_profile(self, settingsFileName):
 			_pJoin(nvdaProfileWorkingDir, "nvda.ini")
 		)
 		# create a package to use as the globalPlugin
-		opSys.copy_directory(
+		opSys.move_directory(
 			_createNvdaSpyPackage(),
 			_pJoin(testSpyPackageDest, systemTestSpyAddonName)
 		)
@@ -118,6 +122,7 @@ def _startNVDAProcess(self):
 		"""Start NVDA.
 		Use debug logging, replacing any current instance, using the system test profile directory
 		"""
+		opSys.file_should_exist(NVDACommandPathToCheckExists, "Unable to start NVDA unless path exists.")
 		self.nvdaHandle = handle = process.start_process(
 			"{baseNVDACommandline} --debug-logging -r -c \"{nvdaProfileDir}\" --log-file \"{nvdaLogFilePath}\"".format(
 				baseNVDACommandline=baseNVDACommandline,
diff --git a/tests/system/readme.md b/tests/system/readme.md
index 4d60af25045..18153d734ef 100644
--- a/tests/system/readme.md
+++ b/tests/system/readme.md
@@ -18,7 +18,13 @@ pip install pyautogui
 
 ### Running the tests
 
-These tests should be run from the windows command prompt (cmd.exe) from the root directory of your NVDA repository.
+These tests should be run from the windows command prompt (cmd.exe) from the root directory
+ of your NVDA repository.
+
+
+```
+python -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:source -P ./tests/system/libraries ./tests/system/
+```
 
 The `whichNVDA` argument allows the tests to be run against an installed copy
 of NVDA (first ensure it is compatible with the tests). Note valid values are:
@@ -26,10 +32,6 @@ of NVDA (first ensure it is compatible with the tests). Note valid values are:
 the tests are run from an administrator command prompt.
 * "source"
 
-```
-python -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:source -P tests/system/libraries tests/system/
-```
-
 To run a single test, use the `--test` argument. Refer to the robot framework documentation for further details.
 
 ```

From 9254b1ec9e9b034dd29cd4a964a009dfa01c1db4 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Thu, 5 Jul 2018 07:28:29 +0000
Subject: [PATCH 87/95] Clean up some of the appveyor code

---
 appveyor.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 73950120257..27bda70c1e0 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -143,9 +143,9 @@ before_test:
 test_script:
  - ps: |
      $errorCode=0
-     $unitTestsXml = (Resolve-Path .\testOutput\unit\unitTests.xml)
-     cd tests
-     py -m nose --with-xunit --xunit-file=$unitTestsXml ./unit
+     $outDir = (Resolve-Path .\testOutput\unit\)
+     $unitTestsXml = "$outDir\unitTests.xml"
+     py -m nose --with-xunit --xunit-file="$unitTestsXml" ./tests/unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      Push-AppveyorArtifact $unitTestsXml
      $wc = New-Object 'System.Net.WebClient'

From edb0e75e1342bf67f0bdcdc9da9cf069a65b4823 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Thu, 5 Jul 2018 08:53:21 +0000
Subject: [PATCH 88/95] clean up some names and docs for tests

---
 tests/system/NVDA Core/startupShutdownNVDA.robot | 3 +++
 tests/system/libraries/nvdaRobotLib.py           | 2 --
 tests/system/readme.md                           | 3 ++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/system/NVDA Core/startupShutdownNVDA.robot b/tests/system/NVDA Core/startupShutdownNVDA.robot
index fdc1d594e64..3398e82794f 100644
--- a/tests/system/NVDA Core/startupShutdownNVDA.robot	
+++ b/tests/system/NVDA Core/startupShutdownNVDA.robot	
@@ -19,9 +19,11 @@ Variables	variables.py
 
 *** Test Cases ***
 Starts
+	[Documentation]	Ensure that NVDA can start
 	process should be running	nvdaAlias
 
 Quits from keyboard
+	[Documentation]	Starts NVDA and ensures that it can be quit using the keyboard
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
 
 	${Welcome dialog title} =	catenate double space	Welcome to NVDA	dialog
@@ -43,6 +45,7 @@ Quits from keyboard
 	process should be stopped	nvdaAlias
 
 Read welcome dialog
+	[Documentation]	Ensure that the welcome dialog can be read in full
 	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
 
 	${Welcome dialog title} =	catenate double space	Welcome to NVDA	dialog
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index faff895d051..45dca03ebd0 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -47,10 +47,8 @@
 
 def _findDepPath(depFileName, searchPaths):
 	import os
-	print searchPaths
 	for path in searchPaths:
 		filePath = _pJoin(path, depFileName+".py")
-		print filePath
 		if os.path.isfile(filePath):
 			return filePath
 		elif os.path.isfile(_pJoin(path, depFileName, "__init__.py")):
diff --git a/tests/system/readme.md b/tests/system/readme.md
index 18153d734ef..b85ada68c97 100644
--- a/tests/system/readme.md
+++ b/tests/system/readme.md
@@ -32,7 +32,8 @@ of NVDA (first ensure it is compatible with the tests). Note valid values are:
 the tests are run from an administrator command prompt.
 * "source"
 
-To run a single test, use the `--test` argument. Refer to the robot framework documentation for further details.
+To run a single test or filter tests, use the `--test` argument (wildcards accepted).
+Refer to the robot framework documentation for further details.
 
 ```
 python -m robot --test "name of test here" ...

From 7bd629ef46331096681d31726f23910b98032096 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Thu, 5 Jul 2018 10:10:19 +0000
Subject: [PATCH 89/95] Add scons command to run system tests

---
 .gitignore                             |  1 +
 readme.md                              | 13 ++++++++
 sconstruct                             |  2 ++
 tests/sconscript                       | 10 +++++-
 tests/system/libraries/nvdaRobotLib.py | 34 +++++++++++++--------
 tests/system/sconscript                | 42 ++++++++++++++++++++++++++
 6 files changed, 88 insertions(+), 14 deletions(-)
 create mode 100644 tests/system/sconscript

diff --git a/.gitignore b/.gitignore
index 1de6c3df665..bec991f09bc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ extras/controllerClient/x64
 source/_buildVersion.py
 user_docs/*/keyCommands.t2t
 output
+testOutput
 source/brailleDisplayDrivers/handyTech
 ./developerGuide.html
 user_docs/build.t2tConf
diff --git a/readme.md b/readme.md
index 51629f54bcc..b6efaadbffd 100644
--- a/readme.md
+++ b/readme.md
@@ -230,3 +230,16 @@ To run only the translatable string checks (which check that all translatable st
 ```
 scons checkPot
 ```
+
+You may also use scons to run the system tests, though this will still rely on having set up the dependencies (see `tests/system/readme.md`).
+
+```
+scons systemTests
+```
+
+To run only specific system tests, specify them using the `filter` variable on the command line.
+This filter accepts wildcard characters.
+
+```
+scons systemTests filter="Read welcome dialog"
+```
\ No newline at end of file
diff --git a/sconstruct b/sconstruct
index 53fa8c0603f..7151932e352 100755
--- a/sconstruct
+++ b/sconstruct
@@ -75,6 +75,8 @@ vars.Add(ListVariable("nvdaHelperDebugFlags", "a list of debugging features you
 vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see, lower is more verbose','15',allowed_values=[str(x) for x in xrange(60)]))
 if "tests" in COMMAND_LINE_TARGETS:
 	vars.Add("unitTests", "A list of unit tests to run", "")
+if "systemTests" in COMMAND_LINE_TARGETS:
+	vars.Add("filter", "A filter for the name of the system test(s) to run. Wildcards accepted.", "")
 
 #Base environment for this and sub sconscripts
 env = Environment(variables=vars,HOST_ARCH='x86',tools=["textfile","gettextTool","t2t",keyCommandsDocTool,'doxygen','recursiveInstall'])
diff --git a/tests/sconscript b/tests/sconscript
index c9450fde5ce..68ff8bad232 100644
--- a/tests/sconscript
+++ b/tests/sconscript
@@ -20,6 +20,11 @@ unitTests = env.SConscript("unit/sconscript", exports=["env"])
 env.Depends(unitTests, sourceDir)
 env.AlwaysBuild(unitTests)
 
+systemTests = env.SConscript("system/sconscript", exports=["env"])
+env.Depends(systemTests, sourceDir)
+env.AlwaysBuild(systemTests)
+env.Alias("systemTests", systemTests)
+
 def checkPotAction(target, source, env):
 	return checkPot.checkPot(source[0].abspath)
 checkPotTarget = env.Command("checkPot", pot, checkPotAction)
@@ -31,10 +36,13 @@ env.Alias("checkPot", checkPotTarget)
 # If specific tests are explicitly specified, only run those.
 explicitUnitTests = env.get("unitTests")
 explicitCheckPot = "checkPot" in COMMAND_LINE_TARGETS
-explicit = explicitUnitTests or explicitCheckPot
+explicitSystemTests = "systemTests" in COMMAND_LINE_TARGETS
+explicit = explicitUnitTests or explicitCheckPot or explicitSystemTests
 tests = []
 if not explicit or explicitUnitTests:
 	tests.append(unitTests)
 if not explicit or explicitCheckPot:
 	tests.append(checkPotTarget)
+if explicit and explicitSystemTests: # only run system tests explicitly
+	tests.append(systemTests)
 env.Alias("tests", tests)
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 45dca03ebd0..66b64c2ae98 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -9,6 +9,7 @@
 """
 # imported methods start with underscore (_) so they don't get imported into robot files as keywords
 from os.path import join as _pJoin, abspath as _abspath, expandvars as _expandvars
+import tempfile
 import sys
 from robotremoteserver import test_remote_server as _testRemoteServer, stop_remote_server as _stopRemoteServer
 from robot.libraries.BuiltIn import BuiltIn
@@ -39,7 +40,9 @@
 
 # Paths
 systemTestSourceDir = _pJoin(repoRoot, "tests", "system")
-nvdaProfileWorkingDir = _pJoin(_expandvars('%TEMP%'), "nvdaProfile")
+tempDir = tempfile.gettempdir()
+opSys.directory_should_exist(tempDir)
+nvdaProfileWorkingDir = _pJoin(tempDir, "nvdaProfile")
 nvdaLogFilePath = _pJoin(nvdaProfileWorkingDir, 'nvda.log')
 systemTestSpyAddonName = "systemTestSpy"
 testSpyPackageDest = _pJoin(nvdaProfileWorkingDir, "globalPlugins")
@@ -65,7 +68,7 @@ def _findDepPath(depFileName, searchPaths):
 def _createNvdaSpyPackage():
 	import os
 	searchPaths = sys.path
-	profileSysTestSpyPackageStagingDir = _pJoin(_expandvars('%TEMP%'), systemTestSpyAddonName)
+	profileSysTestSpyPackageStagingDir = _pJoin(tempDir, systemTestSpyAddonName)
 	# copy in required dependencies, the addon will modify the python path
 	# to point to this sub dir
 	spyPackageLibsDir = _pJoin(profileSysTestSpyPackageStagingDir, "libs")
@@ -87,6 +90,17 @@ def _createNvdaSpyPackage():
 	)
 	return profileSysTestSpyPackageStagingDir
 
+def _createTestIdFileName(name):
+	outDir = builtIn.get_variable_value("${OUTPUT DIR}", )
+	suiteName = builtIn.get_variable_value("${SUITE NAME}")
+	testName = builtIn.get_variable_value("${TEST NAME}")
+	outputFileName = "{suite}-{test}-{name}" \
+		.format(
+		suite=suiteName,
+		test=testName,
+		name=name,
+	).replace(" ", "_")
+	return _pJoin(outDir, "nvdaTestRunLogs", outputFileName)
 
 class nvdaRobotLib(object):
 
@@ -128,11 +142,12 @@ def _startNVDAProcess(self):
 				nvdaLogFilePath=nvdaLogFilePath
 			),
 			shell=True,
-			alias='nvdaAlias'
+			alias='nvdaAlias',
+			stdout=_createTestIdFileName("stdout.txt"),
+			stderr=_createTestIdFileName("stderr.txt"),
 		)
 		return handle
 
-
 	def _connectToRemoteServer(self):
 		"""Connects to the nvdaSpyServer
 		Because we do not know how far through the startup NVDA is, we have to poll
@@ -184,17 +199,10 @@ def start_NVDA(self, settingsFileName):
 	def save_NVDA_log(self):
 		"""NVDA logs are saved to the ${OUTPUT DIR}/nvdaTestRunLogs/${SUITE NAME}-${TEST NAME}-nvda.log"""
 		builtIn.log("saving NVDA log")
-		outDir = builtIn.get_variable_value("${OUTPUT DIR}", )
-		suiteName = builtIn.get_variable_value("${SUITE NAME}")
-		testName = builtIn.get_variable_value("${TEST NAME}")
-		outputFileName = "{suite}-{test}-nvda.log"\
-			.format(
-				suite=suiteName,
-				test=testName,
-			).replace(" ", "_")
+		outputFileName = _createTestIdFileName("nvda.log")
 		opSys.copy_file(
 			nvdaLogFilePath,
-			_pJoin(outDir, "nvdaTestRunLogs", outputFileName)
+			outputFileName
 		)
 
 	def quit_NVDA(self):
diff --git a/tests/system/sconscript b/tests/system/sconscript
new file mode 100644
index 00000000000..c6b800c5fab
--- /dev/null
+++ b/tests/system/sconscript
@@ -0,0 +1,42 @@
+###
+#This file is a part of the NVDA project.
+#URL: http://www.nvaccess.org/
+#Copyright 2017 NV Access Limited.
+#This program is free software: you can redistribute it and/or modify
+#it under the terms of the GNU General Public License version 2.0, as published by
+#the Free Software Foundation.
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#This license can be found at:
+#http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+###
+
+import sys
+Import("env")
+
+import os
+
+tests = env.get("filter")
+env = Environment(
+	ENV={
+		'PATH': os.environ['PATH'],
+		'TEMP': os.environ['TEMP'],
+		'TMP': os.environ['TMP'],
+	})
+
+
+cmd = [
+	sys.executable, "-m", "robot",
+	"--loglevel", "DEBUG",
+	"-d", "testOutput/system",
+	"-x", "systemTests.xml",
+	"-P", "./tests/system/libraries",
+	"-v", "whichNVDA:source",
+]
+if tests:
+	# run specific tests
+	cmd += ['--test="{}"'.format(tests)]
+cmd.append("./tests/system")
+target = env.Command(".", None, [cmd])
+Return('target')
\ No newline at end of file

From 211466434a75271b2b134069c54239278c0295c3 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Thu, 5 Jul 2018 11:49:07 +0000
Subject: [PATCH 90/95] ensure dir is created for std out/err logs

---
 tests/system/libraries/nvdaRobotLib.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index 66b64c2ae98..eab1570b8b1 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -46,6 +46,8 @@
 nvdaLogFilePath = _pJoin(nvdaProfileWorkingDir, 'nvda.log')
 systemTestSpyAddonName = "systemTestSpy"
 testSpyPackageDest = _pJoin(nvdaProfileWorkingDir, "globalPlugins")
+outDir = builtIn.get_variable_value("${OUTPUT DIR}")
+testOutputNvdaLogsDir = _pJoin(outDir, "nvdaTestRunLogs")
 
 
 def _findDepPath(depFileName, searchPaths):
@@ -91,16 +93,14 @@ def _createNvdaSpyPackage():
 	return profileSysTestSpyPackageStagingDir
 
 def _createTestIdFileName(name):
-	outDir = builtIn.get_variable_value("${OUTPUT DIR}", )
 	suiteName = builtIn.get_variable_value("${SUITE NAME}")
 	testName = builtIn.get_variable_value("${TEST NAME}")
-	outputFileName = "{suite}-{test}-{name}" \
-		.format(
+	outputFileName = "{suite}-{test}-{name}".format(
 		suite=suiteName,
 		test=testName,
 		name=name,
 	).replace(" ", "_")
-	return _pJoin(outDir, "nvdaTestRunLogs", outputFileName)
+	return outputFileName
 
 class nvdaRobotLib(object):
 
@@ -134,6 +134,7 @@ def _startNVDAProcess(self):
 		"""Start NVDA.
 		Use debug logging, replacing any current instance, using the system test profile directory
 		"""
+		opSys.create_directory(testOutputNvdaLogsDir)
 		opSys.file_should_exist(NVDACommandPathToCheckExists, "Unable to start NVDA unless path exists.")
 		self.nvdaHandle = handle = process.start_process(
 			"{baseNVDACommandline} --debug-logging -r -c \"{nvdaProfileDir}\" --log-file \"{nvdaLogFilePath}\"".format(
@@ -143,8 +144,8 @@ def _startNVDAProcess(self):
 			),
 			shell=True,
 			alias='nvdaAlias',
-			stdout=_createTestIdFileName("stdout.txt"),
-			stderr=_createTestIdFileName("stderr.txt"),
+			stdout=_pJoin(testOutputNvdaLogsDir, _createTestIdFileName("stdout.txt")),
+			stderr=_pJoin(testOutputNvdaLogsDir, _createTestIdFileName("stderr.txt")),
 		)
 		return handle
 
@@ -199,10 +200,10 @@ def start_NVDA(self, settingsFileName):
 	def save_NVDA_log(self):
 		"""NVDA logs are saved to the ${OUTPUT DIR}/nvdaTestRunLogs/${SUITE NAME}-${TEST NAME}-nvda.log"""
 		builtIn.log("saving NVDA log")
-		outputFileName = _createTestIdFileName("nvda.log")
+		opSys.create_directory(testOutputNvdaLogsDir)
 		opSys.copy_file(
 			nvdaLogFilePath,
-			outputFileName
+			_pJoin(testOutputNvdaLogsDir, _createTestIdFileName("nvda.log"))
 		)
 
 	def quit_NVDA(self):

From f8606bcc172caacc68401acb42df80467bffd391 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Fri, 6 Jul 2018 08:10:05 +0000
Subject: [PATCH 91/95] Use dummy speech synth to get speech

This captures all NVDA manipulation of speech before it is given to
synthesizers.
---
 source/speech.py                              |  5 ----
 tests/system/NVDA Core/variables.py           | 15 +++++-----
 tests/system/libraries/nvdaRobotLib.py        |  8 ++++++
 tests/system/libraries/speechSpy.py           | 28 +++++++++++++++++++
 tests/system/libraries/systemTestSpy.py       | 27 ++++++++++--------
 .../standard-doShowWelcomeDialog.ini          |  2 +-
 .../standard-dontShowWelcomeDialog.ini        |  2 +-
 7 files changed, 60 insertions(+), 27 deletions(-)
 create mode 100644 tests/system/libraries/speechSpy.py

diff --git a/source/speech.py b/source/speech.py
index 1c3b68101b9..14d1988887b 100755
--- a/source/speech.py
+++ b/source/speech.py
@@ -27,10 +27,6 @@
 import speechDictHandler
 import characterProcessing
 import languageHandler
-import extensionPoints
-
-# inform those who want to know that there is new speech
-preSpeech = extensionPoints.Action()
 
 speechMode_off=0
 speechMode_beeps=1
@@ -497,7 +493,6 @@ def speak(speechSequence,symbolLevel=None):
 	"""
 	if not speechSequence: #Pointless - nothing to speak 
 		return
-	preSpeech.notify(speechSequence=speechSequence)
 	import speechViewer
 	if speechViewer.isActive:
 		for item in speechSequence:
diff --git a/tests/system/NVDA Core/variables.py b/tests/system/NVDA Core/variables.py
index 185819ac18f..edf4fc4dc32 100644
--- a/tests/system/NVDA Core/variables.py	
+++ b/tests/system/NVDA Core/variables.py	
@@ -1,14 +1,13 @@
 WELCOME_DIALOG_TEXT = (
-	"Welcome to NVDA  dialog  Welcome to NVDA!\n"
-	"Most commands for controlling NVDA require you to hold down the NVDA key while pressing other keys.\n"
-	"By default, the numpad Insert and main Insert keys may both be used as the NVDA key.\n"
-	"You can also configure NVDA to use the CapsLock as the NVDA key.\n"
-	"Press NVDA+n at any time to activate the NVDA menu.\n"
-	"From this menu, you can configure NVDA, get help and access other NVDA functions.\n"
+	"Welcome to NVDA  dialog  Welcome to NVDA! Most commands for controlling NVDA require you to hold "
+	"down the NVDA key while pressing other keys. By default, the numpad Insert and main Insert keys "
+	"may both be used as the NVDA key. You can also configure NVDA to use the Caps Lock as the NVDA "
+	"key. Press NVDA plus n at any time to activate the NVDA menu. From this menu, you can configure "
+	"NVDA, get help and access other NVDA functions.\n"
 	"Options  grouping\n"
-	"Keyboard layout:  combo box  desktop  collapsed  Alt+k"
+	"Keyboard layout:  combo box  desktop  collapsed  Alt plus k"
 )
 QUIT_DIALOG_TEXT = (
 	"Exit NVDA  dialog\n"
-	"What would you like to do?  combo box  Exit  collapsed  Alt+d"
+	"What would you like to do?  combo box  Exit  collapsed  Alt plus d"
 )
\ No newline at end of file
diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py
index eab1570b8b1..c58964deb19 100644
--- a/tests/system/libraries/nvdaRobotLib.py
+++ b/tests/system/libraries/nvdaRobotLib.py
@@ -119,6 +119,11 @@ def setup_nvda_profile(self, settingsFileName):
 			_createNvdaSpyPackage(),
 			_pJoin(testSpyPackageDest, systemTestSpyAddonName)
 		)
+		# install the test spy speech synth
+		opSys.copy_file(
+			_pJoin(systemTestSourceDir, "libraries", "speechSpy.py"),
+			_pJoin(nvdaProfileWorkingDir, "synthDrivers", "speechSpy.py")
+		)
 
 	def teardown_nvda_profile(self):
 		builtIn.log("Removing files from NVDA profile")
@@ -129,6 +134,9 @@ def teardown_nvda_profile(self):
 			testSpyPackageDest,
 			recursive=True
 		)
+		opSys.remove_file(
+			_pJoin(nvdaProfileWorkingDir, "synthDrivers", "speechSpy.py")
+		)
 
 	def _startNVDAProcess(self):
 		"""Start NVDA.
diff --git a/tests/system/libraries/speechSpy.py b/tests/system/libraries/speechSpy.py
new file mode 100644
index 00000000000..89b29e482f5
--- /dev/null
+++ b/tests/system/libraries/speechSpy.py
@@ -0,0 +1,28 @@
+#A part of NonVisual Desktop Access (NVDA)
+#Copyright (C) 2018 NV Access Limited
+#This file is covered by the GNU General Public License.
+#See the file COPYING for more details.
+
+import synthDriverHandler
+import extensionPoints
+
+# inform those who want to know that there is new speech
+post_speech = extensionPoints.Action()
+
+class SynthDriver(synthDriverHandler.SynthDriver):
+	"""A dummy synth driver used by system tests to get speech output
+	"""
+	name = "speechSpy"
+	description = "System test speech spy"
+
+	@classmethod
+	def check(cls):
+		return True
+
+	supportedSettings = []
+
+	def speak(self, speechSequence):
+		post_speech.notify(speechSequence=speechSequence)
+
+	def cancel(self):
+		pass
diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py
index 1021b32eda4..37af4dfe803 100644
--- a/tests/system/libraries/systemTestSpy.py
+++ b/tests/system/libraries/systemTestSpy.py
@@ -41,9 +41,10 @@ def __init__(self):
 
 	def _registerWithExtensionPoints(self):
 		from core import postNvdaStartup
-		from speech import preSpeech
 		postNvdaStartup.register(self._onNvdaStartupComplete)
-		preSpeech.register(self._onNvdaSpeech)
+
+		from synthDrivers.speechSpy import post_speech
+		post_speech.register(self._onNvdaSpeech)
 
 	# callbacks for extension points
 	def _onNvdaStartupComplete(self):
@@ -151,17 +152,19 @@ def get_last_speech_index(self):
 
 	def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5):
 		sinceIndex = 0 if not sinceIndex else sinceIndex
-		try:
-			success, speechIndex = _blockUntilConditionMet(
-				getValue=lambda: self._spy.getIndexOfSpeech(speech, sinceIndex),
-				giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
-				shouldStopEvaluator=lambda speechIndex: speechIndex >= 0,
-				intervalBetweenSeconds=0.1,
-				errorMessage="Specific speech did not occur before timeout: {}".format(speech)
-			)
-		except AssertionError:
+		success, speechIndex = _blockUntilConditionMet(
+			getValue=lambda: self._spy.getIndexOfSpeech(speech, sinceIndex),
+			giveUpAfterSeconds=self._minTimeout(maxWaitSeconds),
+			shouldStopEvaluator=lambda speechIndex: speechIndex >= 0,
+			intervalBetweenSeconds=0.1,
+			errorMessage=None
+		)
+		if not success:
 			self._spy.dumpSpeechToLog()
-			raise
+			raise AssertionError(
+				"Specific speech did not occur before timeout: {}\n"
+				"See NVDA log for dump of all speech.".format(speech)
+			)
 		return speechIndex
 
 	def wait_for_speech_to_finish(self, maxWaitSeconds=5.0):
diff --git a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
index de69872be1b..e0bada4b1e3 100644
--- a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
@@ -4,4 +4,4 @@ schemaVersion = 2
 [update]
 	askedAllowUsageStats = True
 [speech]
-	synth = silence
+	synth = speechSpy
diff --git a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
index d83ef74cb5a..584e56a6b7d 100644
--- a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
@@ -4,4 +4,4 @@ schemaVersion = 2
 [update]
 	askedAllowUsageStats = True
 [speech]
-	synth = silence
+	synth = speechSpy

From c8b919981c072a4f60114bf3b80bd59c5f966b9d Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Fri, 6 Jul 2018 08:28:44 +0000
Subject: [PATCH 92/95] wait longer before sending input

---
 tests/system/NVDA Core/startupShutdownNVDA.robot | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/system/NVDA Core/startupShutdownNVDA.robot b/tests/system/NVDA Core/startupShutdownNVDA.robot
index 3398e82794f..edd6050aba5 100644
--- a/tests/system/NVDA Core/startupShutdownNVDA.robot	
+++ b/tests/system/NVDA Core/startupShutdownNVDA.robot	
@@ -39,7 +39,7 @@ Quits from keyboard
 	wait for speech to finish
 	${actual speech} =	get speech from index until now	${INDEX}
 	assert strings are equal	${actual speech}	${QUIT_DIALOG_TEXT}
-
+	sleep	1	# the dialog is not always receiving the enter keypress, wait a little longer for it
 	send key	enter
 	wait for process	nvdaAlias	timeout=10 sec
 	process should be stopped	nvdaAlias
@@ -53,4 +53,5 @@ Read welcome dialog
 	wait for speech to finish
 	${actual speech} =	get speech from index until now	${INDEX}
 	assert strings are equal	${actual speech}	${WELCOME_DIALOG_TEXT}
+	sleep	1	# the dialog is not always receiving the enter keypress, wait a little longer for it
 	send key	enter

From 0e743acb1691543f3d5232101ea49ec6455c02d7 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Fri, 6 Jul 2018 09:28:23 +0000
Subject: [PATCH 93/95] Stop update dialog from showing on startup

---
 .../system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini  | 3 +++
 .../nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini       | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
index e0bada4b1e3..be26019b045 100644
--- a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini
@@ -3,5 +3,8 @@ schemaVersion = 2
 	showWelcomeDialogAtStartup = True
 [update]
 	askedAllowUsageStats = True
+	autoCheck = False
+	startupNotification = False
+	allowUsageStats = False
 [speech]
 	synth = speechSpy
diff --git a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
index 584e56a6b7d..9cf3c88beda 100644
--- a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
+++ b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini
@@ -3,5 +3,8 @@ schemaVersion = 2
 	showWelcomeDialogAtStartup = False
 [update]
 	askedAllowUsageStats = True
+	autoCheck = False
+	startupNotification = False
+	allowUsageStats = False
 [speech]
 	synth = speechSpy

From 21eca3c75894de2ace798aff288137f8d251cab9 Mon Sep 17 00:00:00 2001
From: Reef Turner <reef@nvaccess.org>
Date: Fri, 6 Jul 2018 09:29:14 +0000
Subject: [PATCH 94/95] simplify the appveyor pip installs

---
 appveyor.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 3532b1b11f4..8445c52b001 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -66,7 +66,6 @@ install:
  - type ssh_known_hosts >> %userprofile%\.ssh\known_hosts
  - cd ..
  - git submodule update --init
- - py -m pip install robotremoteserver
 
 build_script:
  - ps: |
@@ -116,10 +115,7 @@ build_script:
  - cd ..
 
 before_test:
- - py -m pip install robotframework
- - py -m pip install robotremoteserver
- - py -m pip install pyautogui
- - py -m pip install nose
+ - py -m pip install robotframework robotremoteserver pyautogui nose
  - mkdir testOutput
  - mkdir testOutput\unit
  - mkdir testOutput\system

From be440a96916a353dd42b7f49b152fcde7cd6e267 Mon Sep 17 00:00:00 2001
From: Michael Curran <mick@nvaccess.org>
Date: Tue, 17 Jul 2018 15:48:00 +1000
Subject: [PATCH 95/95] Update What's new

---
 user_docs/en/changes.t2t | 1 +
 1 file changed, 1 insertion(+)

diff --git a/user_docs/en/changes.t2t b/user_docs/en/changes.t2t
index e31d3d559ff..95d6a545e83 100644
--- a/user_docs/en/changes.t2t
+++ b/user_docs/en/changes.t2t
@@ -28,6 +28,7 @@ What's New in NVDA
 
 == Changes for Developers ==
 - Added scriptHandler.script, which can function as a decorator for scripts on scriptable objects. (#6266)
+- A system test framework has been introduced for NVDA. (#708)
 
 
 = 2018.2 =