Add: [Win32] regression testing with MSVC

azure-pipelines-ci.yml

@@ -34,14 +34,25 @@ jobs:
     workingDirectory: $(Build.ArtifactStagingDirectory)
   - script: $(Build.ArtifactStagingDirectory)\windows-dependencies\vcpkg.exe integrate install
     displayName: 'Install dependencies'
+  - bash: |
+      set -ex
+      cd bin/baseset
+      curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
+      unzip opengfx-0.5.2-all.zip
+      rm -f opengfx-0.5.2-all.zip
+    displayName: 'Install OpenGFX'
   - task: VSBuild@1
     displayName: 'Build'
     inputs:
       solution: 'projects/openttd_vs141.sln'
       platform: $(BuildPlatform)
       configuration: Release
       maximumCpuCount: true
-  # Running the regression is currently not possibe via MSVC (console is not redirected)
+  - script: |
+      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86
+      cd projects
+      call regression.bat
+    displayName: 'Test'
 
 - job: linux
   displayName: 'Linux'

bin/ai/regression/run.vbs

@@ -0,0 +1,154 @@
+Option Explicit
+
+' $Id$
+'
+' This file is part of OpenTTD.
+' OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
+' OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+' See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
+
+Dim FSO
+Set FSO = CreateObject("Scripting.FileSystemObject")
+
+Function GetTestList()
+	Dim retests, i, tests, dir
+	Set retests = New RegExp
+	Set GetTestList = CreateObject("Scripting.Dictionary")
+
+	retests.Pattern = "ai/regression/tst_*"
+	retests.Global = True
+	For i = 0 To WScript.Arguments.Count - 1
+		Dim test
+		test = "ai/regression/tst_" & WScript.Arguments.Item(i)
+		If FSO.FolderExists(test) Then
+			retests.Pattern = test
+			Exit For
+		End If
+	Next
+
+	For Each dir In FSO.GetFolder("ai/regression/").SubFolders
+		Dim name
+		name = "ai/regression/" & dir.Name
+		If retests.Test(name) Then
+			GetTestList.Add name, name
+		End If
+	Next
+End Function
+
+Function GetParams()
+	GetParams = "-snull -mnull -vnull:ticks=30000"
+	If WScript.Arguments.Count = 0 Then Exit Function
+	If WScript.Arguments.Item(0) <> "-r" Then Exit Function
+	GetParams = ""
+End Function
+
+Sub FilterFile(filename)
+	Dim lines, filter, file
+
+	Set file = FSO.OpenTextFile(filename, 1)
+	If Not file.AtEndOfStream Then
+		lines = file.ReadAll
+	End If
+	file.Close
+
+	Set filter = New RegExp
+	filter.Global = True
+	filter.Multiline = True
+	filter.Pattern = "0x(\(nil\)|0+)(x0)?"
+	lines = filter.Replace(lines, "0x00000000")
+	filter.Pattern = "^dbg: \[script\]"
+	lines = filter.Replace(lines, "")
+	filter.Pattern = "^ "
+	lines = filter.Replace(lines, "ERROR: ")
+	filter.Pattern = "ERROR: \[1\] \[P\] "
+	lines = filter.Replace(lines, "")
+	filter.Pattern = "^dbg: .*\r\n"
+	lines = filter.Replace(lines, "")
+
+	Set file = FSO.OpenTextFile(filename, 2)
+	file.Write lines
+	file.Close
+End Sub
+
+Function CompareFiles(filename1, filename2)
+	Dim file, lines1, lines2
+	Set file = FSO.OpenTextFile(filename1, 1)
+	If Not file.AtEndOfStream Then
+		lines1 = file.ReadAll
+	End IF
+	file.Close
+	Set file = FSO.OpenTextFile(filename2, 1)
+	If Not file.AtEndOfStream Then
+		lines2 = file.ReadAll
+	End IF
+	file.Close
+	CompareFiles = (lines1 = lines2)
+End Function
+
+Function RunTest(test, params, ret)
+	Dim WshShell, oExec, sav, command
+	Set WshShell = CreateObject("WScript.Shell")
+
+	' Make sure that only one info.nut is present for each test run. Otherwise openttd gets confused.
+	FSO.CopyFile "ai/regression/regression_info.nut", test & "/info.nut"
+
+	sav = test & "/test.sav"
+	If Not FSO.FileExists(sav) Then
+		sav = "ai/regression/empty.sav"
+	End If
+
+	command = ".\openttd -x -c ai/regression/regression.cfg " & params & " -g " & sav & " -d script=2 -d misc=9"
+	' 2>&1 must be after >tmp.regression, else stderr is not redirected to the file
+	WshShell.Run "cmd /c " & command & " >tmp.regression 2>&1", 0, True
+
+	FilterFile "tmp.regression"
+
+	If CompareFiles(test & "/result.txt", "tmp.regression") Then
+		RunTest = "passed!"
+	Else
+		RunTest = "failed!"
+		ret = 1
+	End If
+
+	FSO.DeleteFile test & "/info.nut"
+End Function
+
+On Error Resume Next
+WScript.StdOut.WriteLine ""
+If Err.Number <> 0 Then
+	WScript.Echo "This script must be started with cscript."
+	WScript.Quit 1
+End If
+On Error Goto 0
+
+If Not FSO.FileExists("ai/regression/run.vbs") Then
+	WScript.Echo "Make sure you are in the root of OpenTTD before starting this script."
+	WScript.Quit 1
+End If
+
+If FSO.FileExists("scripts/game_start.scr") Then
+	FSO.MoveFile "scripts/game_start.scr", "scripts/game_start.scr.regression"
+End If
+
+Dim params, test, ret
+params = GetParams()
+ret = 0
+
+For Each test in GetTestList()
+	WScript.StdOut.Write "Running " & test & "... "
+	WScript.StdOut.WriteLine RunTest(test, params, ret)
+Next
+
+If FSO.FileExists("scripts/game_start.scr.regression") Then
+	FSO.MoveFile "scripts/game_start.scr.regression", "scripts/game_start.scr"
+End If
+
+If WScript.Arguments.Count > 0 Then
+	If WScripts.Arguments.Items(0) = "-k" Then
+		WScript.Quit ret
+	End If
+End If
+
+FSO.DeleteFile "tmp.regression"
+
+WScript.Quit ret

projects/openttd_vs140.sln

@@ -24,6 +24,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "settings", "settings_vs140.
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "settingsgen", "settingsgen_vs140.vcxproj", "{E9548DE9-F089-49B7-93A6-30BE2CC311C7}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "regression", "regression_vs140.vcxproj", "{4712B013-437D-42CE-947F-DEBABA15261F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Win32 = Debug|Win32
@@ -84,6 +86,10 @@ Global
 		{E9548DE9-F089-49B7-93A6-30BE2CC311C7}.Release|Win32.Build.0 = Debug|Win32
 		{E9548DE9-F089-49B7-93A6-30BE2CC311C7}.Release|x64.ActiveCfg = Debug|Win32
 		{E9548DE9-F089-49B7-93A6-30BE2CC311C7}.Release|x64.Build.0 = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Debug|Win32.ActiveCfg = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Debug|x64.ActiveCfg = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Release|Win32.ActiveCfg = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Release|x64.ActiveCfg = Debug|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

projects/openttd_vs141.sln

@@ -24,6 +24,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "settings", "settings_vs141.
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "settingsgen", "settingsgen_vs141.vcxproj", "{E9548DE9-F089-49B7-93A6-30BE2CC311C7}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "regression", "regression_vs141.vcxproj", "{4712B013-437D-42CE-947F-DEBABA15261F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Win32 = Debug|Win32
@@ -84,6 +86,10 @@ Global
 		{E9548DE9-F089-49B7-93A6-30BE2CC311C7}.Release|Win32.Build.0 = Debug|Win32
 		{E9548DE9-F089-49B7-93A6-30BE2CC311C7}.Release|x64.ActiveCfg = Debug|Win32
 		{E9548DE9-F089-49B7-93A6-30BE2CC311C7}.Release|x64.Build.0 = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Debug|Win32.ActiveCfg = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Debug|x64.ActiveCfg = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Release|Win32.ActiveCfg = Debug|Win32
+		{4712B013-437D-42CE-947F-DEBABA15261F}.Release|x64.ActiveCfg = Debug|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

projects/regression.bat

@@ -0,0 +1,6 @@
+cd ..\bin
+editbin /nologo /subsystem:console openttd.exe
+cscript /nologo ai\regression\run.vbs
+set RESULT=%ERRORLEVEL%
+editbin /nologo /subsystem:windows openttd.exe
+exit %RESULT%

projects/regression_vs140.vcxproj

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{4712B013-437D-42CE-947F-DEBABA15261F}</ProjectGuid>
+    <RootNamespace>regression</RootNamespace>
+    <ProjectName>regression</ProjectName>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <PlatformToolset>v140</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <NMakeBuildCommandLine>call regression.bat</NMakeBuildCommandLine>
+    <NMakeReBuildCommandLine>call regression.bat</NMakeReBuildCommandLine>
+    <NMakeCleanCommandLine>del ..\bin\tmp.regression</NMakeCleanCommandLine>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="openttd_vs140.vcxproj">
+      <Project>{668328a0-b40e-4cdb-bd72-d0064424414a}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

projects/regression_vs141.vcxproj

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{4712B013-437D-42CE-947F-DEBABA15261F}</ProjectGuid>
+    <RootNamespace>regression</RootNamespace>
+    <ProjectName>regression</ProjectName>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <NMakeBuildCommandLine>call regression.bat</NMakeBuildCommandLine>
+    <NMakeReBuildCommandLine>call regression.bat</NMakeReBuildCommandLine>
+    <NMakeCleanCommandLine>del ..\bin\tmp.regression</NMakeCleanCommandLine>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="openttd_vs141.vcxproj">
+      <Project>{668328a0-b40e-4cdb-bd72-d0064424414a}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

src/os/windows/win32.cpp

@@ -300,7 +300,7 @@ void CreateConsole()
 	if (_has_console) return;
 	_has_console = true;
 
-	AllocConsole();
+	if (!AllocConsole()) return;
 
 	hand = GetStdHandle(STD_OUTPUT_HANDLE);
 	GetConsoleScreenBufferInfo(hand, &coninfo);