Build and run damlc on Windows (#568)

* Fix network build on Windows

Some files were not added to the build, which led missing symbols at
link time.

* Drop dll.a files from Windows GHC bindist

Those files greatly confuse GHC when linking statically.

* Add some Windows system libraries

These libraries are needed when linking GRPC.

* Statically link pthread on Windows

Otherwise the executables fail at runtime because they cannot find the
shared object.

* Build and run damlc on CI

* Try to fix package_db/* nullglob error

* Fix powershell command

* Cleanup package db rule

* Make formatting ugly again

3rdparty/haskell/network-package.bzl

@@ -73,7 +73,7 @@ struct(
           extraFrameworkDirs = [ ],
           asmSources = [ ],
           cmmSources = [ ],
-          cSources = [ "cbits/ancilData.c", "cbits/HsNet.c", ],
+          cSources = [ "cbits/ancilData.c", "cbits/HsNet.c", "cbits/initWinSock.c", "cbits/winSockErr.c", "cbits/asyncAccept.c" ],
           cxxSources = [ ],
           jsSources = [ ],
           hsSourceDirs = [ ],

bazel_tools/BUILD.bazel

@@ -10,6 +10,7 @@ exports_files(
         "haskell-static-linking.patch",
         "haskell-arx.patch",
         "haskell-win-sys-includes.patch",
+        "haskell-drop-fake-static.patch",
         "pom_template.xml",
     ],
     visibility = ["//:__subpackages__"],

bazel_tools/haskell-drop-fake-static.patch

@@ -0,0 +1,18 @@
+The presence of files with `dll.a` extensions greatly confuses GHC, so we
+remove them for the libraries we link in.
+
+diff --git a/haskell/ghc_bindist.bzl b/haskell/ghc_bindist.bzl
+index 941bf18..f9f3276 100644
+--- a/haskell/ghc_bindist.bzl
++++ b/haskell/ghc_bindist.bzl
+@@ -201,6 +201,10 @@ def _ghc_bindist_impl(ctx):
+         _execute_fail_loudly(ctx, ["./configure", "--prefix", bindist_dir.realpath])
+         _execute_fail_loudly(ctx, ["make", "install"])
+
++    if os == "windows":
++        _execute_fail_loudly(ctx, ["rm", "mingw/lib/gcc/x86_64-w64-mingw32/7.2.0/libstdc++.dll.a"])
++        _execute_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libwinpthread.dll.a"])
++
+     ctx.template(
+         "BUILD",
+         ghc_build,

bazel_tools/haskell-static-linking.patch

@@ -1,5 +1,9 @@
 This is required to get the GHCi/TH linker to pick up static Haskell libs.
 Buck uses the same trick. On Windows we retain the original behavior.
+
+Moreover on Windows we add extra system dependencies for grpc. The solution is
+not ideal but works well; for a proper solution we need better system library
+support in rules_haskell: https://github.com/tweag/rules_haskell/issues/834
 diff --git a/haskell/haskell.bzl b/haskell/haskell.bzl
 index 7d26a6d..e2a289b 100644
 --- a/haskell/haskell.bzl
@@ -32,7 +36,7 @@ index 2163c02..37a6d07 100644
 
 +    metadata_entries_extras = {
 +        "hs-libraries": pkg_id.library_name(hs, my_pkg_id),
-+        "extra-libraries": " ".join(extra_libs),
++        "extra-libraries": " ".join(extra_libs + (["stdc++", "crypt32", "ws2_32"] if my_pkg_id.name == "grpc-haskell-core" else [])),
 +        } if hs.toolchain.is_windows else {
 +        "extra-libraries": " ".join([pkg_id.library_name(hs, my_pkg_id)] + extra_libs),
 +    }

build.ps1

@@ -63,6 +63,7 @@ function build-full() {
         //daml-lf/transaction-scalacheck/... `
         //daml-lf/validation/... `
         //daml-foundations/daml-tools/docs/... `
+        //daml-foundations/daml-tools/da-hs-damlc-app `
         //language-support/java/testkit:testkit `
         //language-support/java/bindings/... `
         //language-support/java/bindings-rxjava/... `
@@ -82,6 +83,14 @@ function build-full() {
     # which is a workaround for this problem.
     bazel shutdown
 
+    bazel run `
+        //daml-foundations/daml-tools/da-hs-damlc-app `-`- `-h
+
+    # ScalaCInvoker, a Bazel worker, created by rules_scala opens some of the bazel execroot's files,
+    # which later causes issues on Bazel init (source forest creation) on Windows. A shutdown closes workers,
+    # which is a workaround for this problem.
+    bazel shutdown
+
     bazel test `
         //daml-lf/data/... `
         //daml-lf/interface/... `

daml-foundations/daml-ghc/package-database/util.bzl

@@ -158,16 +158,20 @@ def _daml_package_db_impl(ctx):
         outputs = [ctx.outputs.tar],
         command =
             """
-      mkdir package_db
-    """ +
+        set -eoux pipefail
+        shopt -s nullglob
+        TMP_DIR=$(mktemp -d)
+        PACKAGE_DB="$TMP_DIR/package_db"
+        mkdir -p "$PACKAGE_DB"
+        """ +
             "".join(
                 [
                     """
-       mkdir -p package_db/{daml_lf_version}/{pkg_name}
-       cp {pkg_conf} package_db/{daml_lf_version}/{pkg_name}.conf
-       tar xf {iface_tar} --strip-components=1 -C package_db/{daml_lf_version}/{pkg_name}/
-       cp {dalf} package_db/{daml_lf_version}/{pkg_name}.dalf
-       """.format(
+        mkdir -p "$PACKAGE_DB/{daml_lf_version}/{pkg_name}"
+        cp {pkg_conf} "$PACKAGE_DB/{daml_lf_version}/{pkg_name}.conf"
+        tar xf {iface_tar} --strip-components=1 -C "$PACKAGE_DB/{daml_lf_version}/{pkg_name}/"
+        cp {dalf} "$PACKAGE_DB/{daml_lf_version}/{pkg_name}.dalf"
+        """.format(
                         daml_lf_version = pkg[DamlPackage].daml_lf_version,
                         pkg_name = pkg[DamlPackage].pkg_name,
                         pkg_conf = pkg[DamlPackage].pkg_conf.path,
@@ -178,11 +182,11 @@ def _daml_package_db_impl(ctx):
                 ],
             ) +
             """
-      for lf_version in package_db/*; do
-        {ghc_pkg} recache --package-db=$lf_version --no-expand-pkgroot
-      done
-      tar cf {db_tar} package_db
-    """.format(
+        for lf_version in "$PACKAGE_DB"/*; do
+          {ghc_pkg} recache --package-db=$lf_version --no-expand-pkgroot
+        done
+        tar cf {db_tar} -C "$TMP_DIR" package_db
+        """.format(
                 db_tar = ctx.outputs.tar.path,
                 ghc_pkg = toolchain.tools.ghc_pkg.path,
             ),

daml-foundations/daml-tools/da-hs-damlc-app/BUILD.bazel

@@ -8,6 +8,13 @@ load("@os_info//:os_info.bzl", "is_windows")
 da_haskell_binary(
     name = "da-hs-damlc-app",
     srcs = ["src/Main.hs"],
+
+    # We need to tell the linker to statically link pthread on Windows
+    # otherwise the library is not found at runtime.
+    compiler_flags = [
+        "-optl-static",
+        "-optl-pthread",
+    ] if is_windows else [],
     data = [
         "//compiler/scenario-service/server:scenario_service_jar",
         "//daml-foundations/daml-ghc/package-database:package-db",
@@ -26,6 +33,13 @@ da_haskell_binary(
 da_haskell_binary(
     name = "damlc_bootstrap",
     srcs = ["src/Main.hs"],
+
+    # We need to tell the linker to statically link pthread on Windows
+    # otherwise the library is not found at runtime.
+    compiler_flags = [
+        "-optl-static",
+        "-optl-pthread",
+    ] if is_windows else [],
     hazel_deps = [
         "base",
     ],

deps.bzl

@@ -43,6 +43,7 @@ def daml_deps():
             patches = [
                 "@com_github_digital_asset_daml//bazel_tools:haskell-static-linking.patch",
                 "@com_github_digital_asset_daml//bazel_tools:haskell-win-sys-includes.patch",
+                "@com_github_digital_asset_daml//bazel_tools:haskell-drop-fake-static.patch",
             ],
             patch_args = ["-p1"],
             sha256 = rules_haskell_sha256,