Merge pull request #10231 from rainers/test_debug

fix building tests with debug compiler
merged-on-behalf-of: Nicholas Wilson <thewilsonator@users.noreply.github.com>

.azure-pipelines/lib.sh

@@ -28,13 +28,33 @@ download() {
     curl -fsSL -A "$CURL_USER_AGENT" --connect-timeout 5 --speed-time 30 --speed-limit 1024 --retry 5 --retry-delay 5 "$url" -o "$path"
 }
 
-install_grep() {
-    local tools_dir="${DMD_DIR}/tools"
-    mkdir -p "$tools_dir"
-    cd "$tools_dir"
-    download "http://downloads.dlang.org/other/grep-3.1.zip" "grep-3.1.zip"
-    unzip "grep-3.1.zip" # contains grep.exe
-    export PATH="${tools_dir}:$PATH"
+################################################################################
+# Download dmd
+################################################################################
+
+install_host_dmd() {
+    if [ ! -f dmd2/README.TXT ]; then
+        download "http://downloads.dlang.org/releases/2.x/${HOST_DMD_VERSION}/dmd.${HOST_DMD_VERSION}.windows.7z" dmd2.7z
+        7z x dmd2.7z > /dev/null
+        download "https://ci.appveyor.com/api/buildjobs/nogriv1wq32h4jr0/artifacts/libcurl-7.65.3-WinSSL-zlib-x86-x64.zip" libcurl.zip
+        7z -y x libcurl.zip > /dev/null
+    fi
+    export PATH="$PWD/dmd2/windows/bin/:$PATH"
+    export HOST_DC="$PWD/dmd2/windows/bin/dmd.exe"
+    export DM_MAKE="$PWD/dmd2/windows/bin/make.exe"
+    dmd --version
+}
+
+################################################################################
+# Download dmc
+################################################################################
+
+install_host_dmc() {
+    if [ ! -f dm/README.TXT ]; then
+        download "http://downloads.dlang.org/other/dm857c.zip" dmc.zip
+        7z x dmc.zip > /dev/null
+    fi
+    dm/bin/dmc | head -n 1 || true
 }
 
 ################################################################################

.azure-pipelines/windows-visual-studio.sh

@@ -53,8 +53,13 @@ dm_make() {
     cp "$DMC" "$DM_MAKE" "dm/path"
 }
 
-echo "[STEP]: Downloading LDC"
-ldc
+if [ "$D_COMPILER" == "dmd" ]; then
+    echo "[STEP]: Downloading DMD"
+    install_host_dmd
+elif [ "$D_COMPILER" == "ldc" ]; then
+    echo "[STEP]: Downloading LDC"
+    ldc
+fi
 
 echo "[STEP]: Downloading VisualD"
 visuald

.azure-pipelines/windows-visual-studio.yml

@@ -14,9 +14,16 @@ steps:
       @echo on
       call "%VSINSTALLDIR%\VC\Auxiliary\Build\vcvarsall.bat" %ARCH%
       @echo on
-      set VISUALD_INSTALLER=VisualD-%VISUALD_VER%.exe
+
       set DMD_DIR=%cd%
-      set DMD=%DMD_DIR%\generated\Windows\Release\Win32\dmd.exe
+      if "%CONFIGURATION%" == "" set CONFIGURATION=Release
+      set PLATFORM=Win32
+      set MODEL=32mscoff
+      if "%ARCH%"=="x64" set PLATFORM=x64
+      if "%ARCH%"=="x64" set MODEL=64
+      set DMD=%DMD_DIR%\generated\Windows\%CONFIGURATION%\%PLATFORM%\dmd.exe
+
+      set VISUALD_INSTALLER=VisualD-%VISUALD_VER%.exe
       set DMD_TESTSUITE_MAKE_ARGS=-j3
       set DM_MAKE=%DMD_DIR%\dm\path\make.exe
       set LDC_DIR=%DMD_DIR%\ldc2-%LDC_VERSION%-windows-multilib
@@ -47,35 +54,40 @@ steps:
 
       .\%VISUALD_INSTALLER% /S
       REM configure DMD path
-      REM reg add "HKLM\SOFTWARE\DMD" /v InstallationFolder /t REG_SZ /d "%DMD_DIR%\dmd2" /reg:32 /f
+      if "%D_COMPILER%" == "dmd" reg add "HKLM\SOFTWARE\DMD" /v InstallationFolder /t REG_SZ /d "%DMD_DIR%" /reg:32 /f
       REM configure LDC path
-      reg add "HKLM\SOFTWARE\LDC" /v InstallationFolder /t REG_SZ /d "%LDC_DIR%" /reg:32 /f
+      if "%D_COMPILER%" == "ldc" reg add "HKLM\SOFTWARE\LDC" /v InstallationFolder /t REG_SZ /d "%LDC_DIR%" /reg:32 /f
 
       REM build via VS projects with LDC
       cd src
-      set LDC_ARGS=%LDC_ARGS% /p:DCompiler=LDC
-      msbuild /target:dmd /p:Configuration=Release /p:Platform=Win32 %LDC_ARGS% vcbuild\dmd.sln || exit /B 1
+      if "%D_COMPILER%" == "ldc" set LDC_ARGS=%LDC_ARGS% /p:DCompiler=LDC
+      msbuild /target:dmd /p:Configuration=%CONFIGURATION% /p:Platform=%PLATFORM% %LDC_ARGS% vcbuild\dmd.sln || exit /B 1
 
       %DMD% --version
       grep --version
 
       REM Check: run druntime unittests
       cd "%DMD_DIR%\..\druntime"
-      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" target || exit /B 2
+      "%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" target || exit /B 2
       echo "[DRUNTIME] running tests..."
-      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" unittest32mscoff || exit /B 3
-      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" test_all || exit /B 4
+      "%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" unittest || exit /B 3
+      "%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" test_all || exit /B 4
 
       REM Check: build phobos
       cd "%DMD_DIR%\..\phobos"
-      "%DM_MAKE%" -f win64.mak MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "AR=%MSVC_AR%" "MAKE=%DM_MAKE%" || exit /B 5
+      "%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "AR=%MSVC_AR%" "MAKE=%DM_MAKE%" || exit /B 5
+
+      REM Build DMD VERSION + string imports (not built by VisualD)
+      copy "%DMD_DIR%\VERSION" "%DMD_DIR%\generated\Windows\Release\Win32\VERSION"
 
       REM Run DMD testsuite
       cd "%DMD_DIR%\test"
-      cp %DMD_DIR%\..\phobos\phobos32mscoff.lib .
-      "%GNU_MAKE%" -j%NUMBER_OF_PROCESSORS% all MODEL=32mscoff ARGS="-O -inline -g" OS=windows DMD="%DMD%" "CC=%MSVC_CC%" DMD_MODEL=win32 || exit /B 6
+      cp %DMD_DIR%\..\phobos\phobos%MODEL%.lib .
+      "%GNU_MAKE%" -j%NUMBER_OF_PROCESSORS% all MODEL=%MODEL% ARGS="-O -inline -g" OS=windows DMD="%DMD%" "CC=%MSVC_CC%" DMD_MODEL=%PLATFORM% BUILD=%CONFIGURATION% || exit /B 6
 
       cd "%DMD_DIR%\..\phobos"
       REM Check: build phobos unittests
-      cp %LDC_DIR%\lib32\libcurl.dll .
-      "%DM_MAKE%" -f win64.mak unittest MODEL=32mscoff "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%"
+      if "%D_COMPILER%" == "ldc" cp %LDC_DIR%\lib32\libcurl.dll .
+      if "%D_COMPILER%_%MODEL%" == "dmd_64" cp %DMD_DIR%\dmd2\windows\bin64\libcurl.dll .
+      if "%D_COMPILER%_%MODEL%" == "dmd_32mscoff" cp %DMD_DIR%\dmd2\windows\bin\libcurl.dll .
+      "%DM_MAKE%" -f win64.mak unittest MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%"

.azure-pipelines/windows.sh

@@ -8,25 +8,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 GNU_MAKE="$(which make)"
 
-install_host_dmd() {
-    if [ ! -f dmd2/README.TXT ]; then
-        download "http://downloads.dlang.org/releases/2.x/${HOST_DMD_VERSION}/dmd.${HOST_DMD_VERSION}.windows.7z" dmd2.7z
-        7z x dmd2.7z > /dev/null
-    fi
-    export PATH="$PWD/dmd2/windows/bin/:$PATH"
-    export HOST_DC="$PWD/dmd2/windows/bin/dmd.exe"
-    export DM_MAKE="$PWD/dmd2/windows/bin/make.exe"
-    dmd --version
-}
-
-install_host_dmc() {
-    if [ ! -f dm/README.TXT ]; then
-        download "http://downloads.dlang.org/other/dm857c.zip" dmc.zip
-        7z x dmc.zip > /dev/null
-    fi
-    dm/bin/dmc | head -n 1 || true
-}
-
 ################################################################################
 # Setup required tools
 ################################################################################

azure-pipelines.yml

@@ -40,7 +40,7 @@ jobs:
       VSINSTALLDIR: C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\
     strategy:
       matrix:
-        visuald-ldc:
+        win32-ldc:
           OS: Win_32
           MODEL: 32
           ARCH: x86
@@ -49,3 +49,22 @@ jobs:
           LDC_VERSION: 1.14.0
     steps:
       - template: .azure-pipelines/windows-visual-studio.yml
+
+  - job: Windows_VisualD_Debug
+    timeoutInMinutes: 120
+    pool:
+      vmImage: 'vs2017-win2016'
+    variables:
+      VSINSTALLDIR: C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\
+      HOST_DMD_VERSION: 2.086.1
+    strategy:
+      matrix:
+        x64-debug-dmd:
+          OS: Win_64
+          MODEL: 64
+          ARCH: x64
+          CONFIGURATION: Debug
+          D_COMPILER: dmd
+          VISUALD_VER: v0.49.0
+    steps:
+      - template: .azure-pipelines/windows-visual-studio.yml

src/dmd/dmangle.d

@@ -586,10 +586,14 @@ public:
         {
             const slice = buf.peekSlice();
             assert(slice.length);
-            foreach (const char c; slice)
+            for (size_t pos; pos < slice.length; )
             {
+                dchar c;
+                auto ppos = pos;
+                auto p = utf_decodeChar(slice.ptr, slice.length, pos, c);
+                assert(p is null, p[0..strlen(p)]);
                 assert(c.isValidMangling, "The mangled name '" ~ slice ~ "' " ~
-                    "contains an invalid character: " ~ c);
+                    "contains an invalid character: " ~ slice[ppos..pos]);
             }
         }
     }
@@ -1110,7 +1114,8 @@ package bool isValidMangling(dchar c) nothrow
         c >= 'A' && c <= 'Z' ||
         c >= 'a' && c <= 'z' ||
         c >= '0' && c <= '9' ||
-        c != 0 && strchr("$%().:?@[]_", c);
+        c != 0 && strchr("$%().:?@[]_", c) ||
+        isUniAlpha(c);
 }
 
 // valid mangled characters

src/dmd/dsymbolsem.d

@@ -3616,7 +3616,9 @@ private extern(C++) final class DsymbolSemanticVisitor : Visitor
                             // a C++ dtor gets its vtblIndex later (and might even be added twice to the vtbl),
                             // e.g. when compiling druntime with a debug compiler, namely with core.stdcpp.exception.
                             if (auto fd = s.isFuncDeclaration())
-                                assert(fd.vtblIndex == i || (cd.classKind == ClassKind.cpp && fd.isDtorDeclaration));
+                                assert(fd.vtblIndex == i ||
+                                       (cd.classKind == ClassKind.cpp && fd.isDtorDeclaration) ||
+                                       funcdecl.parent.isInterfaceDeclaration); // interface functions can be in multiple vtbls
                         }
                     }
                     else

src/vcbuild/dmd.vcxproj

@@ -221,7 +221,7 @@
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\..\VERSION">
-      <Message>Creating VERSION...</Message>
+      <Message>Creating VERSION: $(_rdmdExe) ..\..\config.d $(OutDir) ..\..\VERSION</Message>
       <Command>$(_rdmdExe) ..\..\config.d $(OutDir) ..\..\VERSION</Command>
       <Outputs>$(OutDir)VERSION;%(Outputs)</Outputs>
     </CustomBuild>

test/dshell/sameenv.d

@@ -8,8 +8,8 @@ void main()
     run("$OUTPUT_BASE/printenv$EXE", File(envFromExe, "wb"));
     run("$DMD -m$MODEL -run $EXTRA_FILES/printenv.d", File(envFromRun, "wb"));
 
-    const fromExe = readText(envFromExe);
-    const fromRun = readText(envFromRun);
+    const fromExe = readText(envFromExe).filterCompilerOutput;
+    const fromRun = readText(envFromRun).filterCompilerOutput;
     if (fromExe != fromRun)
     {
         writefln("FromExe:");

test/fail_compilation/fail7524a.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # https://issues.dlang.org/show_bug.cgi?id=7524
 
-output="$(echo '#line 47 __DATE__' | (! "$DMD" -c -o- - 2>&1))"
+output="$(echo '#line 47 __DATE__' | (! "$DMD" -c -o- - 2>&1) | grep -v 'DEBUG\|DMD')"
 
 line1='__stdin.d(1): Error: #line integer ["filespec"]\n expected'
 if [ "$(echo "$output" | head -n1 | tr -d "\r")"  != "$line1" ] ; then

test/runnable/extra-files/statictor-postscript.sh

@@ -2,7 +2,7 @@
 
 # trim off the first line which contains the path of the file which differs between windows and non-windows
 # also trim off compiler debug message and remove CR
-grep -v "runnable\|DEBUG" $1 | tr -d "\r" > ${OUTPUT_BASE}.d.out.2
+grep -v "runnable\|DEBUG\|DMD" $1 | tr -d "\r" > ${OUTPUT_BASE}.d.out.2
 
 diff -pu --strip-trailing-cr ${EXTRA_FILES}/${TEST_NAME}.d.out ${OUTPUT_BASE}.d.out.2
 

test/runnable/extra-files/test17868-postscript.sh

@@ -3,7 +3,7 @@
 # trim off the first line which contains the path of the file which differs between windows and non-windows
 # also trim off compiler debug message and remove CR
 echo ${OUTPUT_BASE}.out.trim
-grep -v 'runnable\|DEBUG' $1 | tr -d "\r" > ${OUTPUT_BASE}.out.trim
+grep -v 'runnable\|DEBUG\|DMD' $1 | tr -d "\r" > ${OUTPUT_BASE}.out.trim
 
 diff -pu --strip-trailing-cr ${EXTRA_FILES}/test17868.d.out ${OUTPUT_BASE}.out.trim
 

test/runnable/test18772.d

@@ -8,5 +8,6 @@ void main()
     cfloat[1] A;
     float[1] B;
     int i = 0;
+    version(D_LP64) {} else // disabled because of wrong codegen: https://issues.dlang.org/show_bug.cgi?id=20089
     double C = fun(A[i] * B[i]);
 }

test/tools/d_do_test.d

@@ -803,7 +803,7 @@ int tryMain(string[] args)
             }
 
             compile_output = compile_output.unifyNewLine();
-            compile_output = std.regex.replace(compile_output, regex(`^DMD v2\.[0-9]+.*\n? DEBUG$`, "m"), "");
+            compile_output = std.regex.replaceAll(compile_output, regex(`^DMD v2\.[0-9]+.*\n? DEBUG$`, "m"), "");
             compile_output = std.string.strip(compile_output);
             // replace test_result path with fixed ones
             compile_output = compile_output.replace(result_path, resultsDirReplacement);

test/tools/dshell_prebuilt/dshell_prebuilt.d

@@ -14,6 +14,7 @@ public import std.string;
 public import std.format;
 public import std.path;
 public import std.file;
+public import std.regex;
 public import std.stdio;
 public import std.process;
 
@@ -274,7 +275,6 @@ GrepResult grep(GrepResult lastResult, string pattern)
 
 private GrepResult grepLines(T)(T lineRange, string finalPattern)
 {
-    import std.regex;
     auto matches = appender!(string[])();
     foreach(line; lineRange)
     {
@@ -289,3 +289,13 @@ private GrepResult grepLines(T)(T lineRange, string finalPattern)
     writefln("[GREP] matched %s lines", matches.data.length);
     return GrepResult(matches.data);
 }
+
+/**
+remove \r and the compiler debug header from the given string.
+*/
+string filterCompilerOutput(string output)
+{
+    output = std.string.replace(output, "\r", "");
+    output = std.regex.replaceAll(output, regex(`^DMD v2\.[0-9]+.*\n? DEBUG\n`, "m"), "");
+    return output;
+}