Native Windows urbit binaries using MingW

.github/workflows/build.yml

@@ -116,3 +116,24 @@ jobs:
       - run: nix-build -A hs.urbit-king.components.exes.urbit-king --arg enableStatic true
       - run: nix-build -A hs-checks
       - run: nix-build shell.nix
+
+  mingw:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: C:\msys64\msys2_shell.cmd -mingw64 -defterm -no-start -here -c ". <(cygpath '{0}')"
+        working-directory: ./pkg/urbit
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          lfs: true
+
+      # echo suppresses pacman prompt
+      - run: echo|./configure
+        env:
+          CACHIX_CACHE: locpyl-tidnyd-test1
+          CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - run: make build/urbit build/urbit-worker
+      - run: build/urbit -l -d -B ../../bin/solid.pill -F bus && curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' http://localhost:12321

.gitignore

@@ -54,6 +54,7 @@ release/
 dist/
 out/
 work/
+pkg/*/*.a
 *.o
 
 # Landscape Dev

nix/sources-pmnsh.json

@@ -0,0 +1,90 @@
+{
+    "curl": {
+        "branch": "master",
+        "description": "A command line tool and library for transferring data with URL syntax",
+        "homepage": "http://curl.se/",
+        "pmnsh": {
+            "include": "include",
+            "lib": "lib/.libs",
+            "prepare": "autoreconf -vfi && ./configure --disable-shared --disable-ldap --disable-rtsp --without-brotli --without-libidn2 --without-libpsl --without-nghttp2 --with-openssl",
+            "make": "-C lib libcurl.la"
+        },
+        "owner": "curl",
+        "repo": "curl",
+        "rev": "curl-7_77_0",
+        "type": "tarball",
+        "url": "https://github.com/curl/curl/archive/curl-7_77_0.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "lmdb": {
+        "branch": "mdb.master",
+        "description": "LMDB library",
+        "homepage": "http://www.lmdb.tech/",
+        "pmnsh": {
+            "strip": 2,
+            "make": "liblmdb.a"
+        },
+        "owner": "LMDB",
+        "repo": "lmdb",
+        "rev": "48a7fed59a8aae623deff415dda27097198ca0c1",
+        "type": "tarball",
+        "url": "https://github.com/LMDB/lmdb/archive/48a7fed59a8aae623deff415dda27097198ca0c1.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "secp256k1": {
+        "branch": "master",
+        "description": "Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1.",
+        "homepage": null,
+        "pmnsh": {
+            "include": "include",
+            "lib": ".libs",
+            "prepare": "./autogen.sh && ./configure --disable-shared --enable-module-recovery CFLAGS=-DSECP256K1_API=",
+            "make": "libsecp256k1.la"
+        },
+        "owner": "bitcoin-core",
+        "repo": "secp256k1",
+        "rev": "26de4dfeb1f1436dae1fcf17f57bdaa43540f940",
+        "type": "tarball",
+        "url": "https://github.com/bitcoin-core/secp256k1/archive/26de4dfeb1f1436dae1fcf17f57bdaa43540f940.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "uv": {
+        "branch": "v1.x",
+        "description": "Cross-platform asynchronous I/O",
+        "homepage": "http://libuv.org/",
+        "pmnsh": {
+            "include": "include",
+            "lib": ".libs",
+            "prepare": "./autogen.sh && ./configure --disable-shared",
+            "make": "libuv.la",
+            "compat": {
+                "m1brew": false
+            }
+        },
+        "owner": "libuv",
+        "repo": "libuv",
+        "rev": "v1.40.0",
+        "type": "tarball",
+        "url": "https://github.com/libuv/libuv/archive/v1.40.0.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "ent": {
+        "pmnsh": {
+            "prepare": "./configure"
+        }
+    },
+    "ge-additions": {
+        "pmnsh": {
+            "make": "CFLAGS=-I../ed25519"
+        }
+    },
+    "libaes_siv": {
+        "pmnsh": {
+            "compat": {
+                "m1brew": {
+                    "make": "CFLAGS=$(pkg-config --cflags openssl)"
+                }
+            }
+        }
+    }
+}

nix/sources.json

@@ -3,18 +3,26 @@
         "branch": "master",
         "description": "With argon2u. Based off https://github.com/P-H-C/phc-winner-argon2",
         "homepage": "",
+        "pmnsh": {
+            "include": ["include", "src/blake2"],
+            "make": "libargon2.a"
+        },
         "owner": "urbit",
         "repo": "argon2",
-        "rev": "4da94a611ee62bad87ab2b131ffda3bcc0723d9c",
-        "sha256": "0bqq1hg367l4jkb6cqhxlblpvdbwz3l586qsfakwzfd9wdvnm3yc",
+        "rev": "a4c1e3f7138c2e577376beb99f964cf71e1c8b1b",
+        "sha256": "1j8a15fx2kn5aa3scmb5qnsfk627kfvsq5ppz9j0pv2d1xck527x",
         "type": "tarball",
-        "url": "https://github.com/urbit/argon2/archive/4da94a611ee62bad87ab2b131ffda3bcc0723d9c.tar.gz",
+        "url": "https://github.com/urbit/argon2/archive/a4c1e3f7138c2e577376beb99f964cf71e1c8b1b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "ed25519": {
         "branch": "master",
         "description": "Submodule included by Urbit",
         "homepage": null,
+        "pmnsh": {
+            "strip": 1,
+            "make": "all"
+        },
         "owner": "urbit",
         "repo": "ed25519",
         "rev": "76385f2ebbbc9580a9c236952d68d11d73a6135c",
@@ -27,12 +35,22 @@
         "branch": "master",
         "description": "H2O - the optimized HTTP/1, HTTP/2, HTTP/3 server",
         "homepage": "https://h2o.examp1e.net",
+        "pmnsh": {
+             "include": "include",
+             "prepare": "cmake .",
+             "make": "libh2o",
+             "compat": {
+                 "mingw": {
+                      "prepare": "cmake -G\"MSYS Makefiles\" ."
+                 }
+             }
+        },
         "owner": "h2o",
         "repo": "h2o",
-        "rev": "v2.2.4",
-        "sha256": "0176x0bzjry19zs074a9i5vhncc842xikmx43wj61jky318nq4w4",
+        "rev": "v2.2.6",
+        "sha256": "0qni676wqvxx0sl0pw9j0ph7zf2krrzqc1zwj73mgpdnsr8rsib7",
         "type": "tarball",
-        "url": "https://github.com/h2o/h2o/archive/v2.2.4.tar.gz",
+        "url": "https://github.com/h2o/h2o/archive/v2.2.6.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "hackage.nix": {
@@ -63,6 +81,9 @@
         "branch": "master",
         "description": null,
         "homepage": null,
+        "pmnsh": {
+            "make": "libscrypt.a CFLAGS_EXTRA=-ffast-math"
+        },
         "owner": "urbit",
         "repo": "libscrypt",
         "rev": "029693ff1cbe4f69d3a2da87d0f4f034f92cc0c2",
@@ -75,6 +96,9 @@
         "branch": "master",
         "description": null,
         "homepage": null,
+        "pmnsh": {
+            "make": "static"
+        },
         "owner": "urbit",
         "repo": "murmur3",
         "rev": "71a75d57ca4e7ca0f7fc2fd84abd93595b0624ca",
@@ -111,6 +135,19 @@
         "branch": "master",
         "description": null,
         "homepage": null,
+        "pmnsh": {
+            "include": "source/include",
+            "compat": {
+                "m1brew": {
+                    "lib": "build/template-FAST_INT64",
+                    "make": "-C build/template-FAST_INT64 libsoftfloat3.a"
+                },
+                "mingw": {
+                    "lib": "build/Win64-MinGW-w64",
+                    "make": "-C build/Win64-MinGW-w64 libsoftfloat3.a"
+                }
+            }
+        },
         "owner": "urbit",
         "repo": "berkeley-softfloat-3",
         "rev": "ec4c7e31b32e07aad80e52f65ff46ac6d6aad986",

pkg/ent/configure

@@ -7,6 +7,7 @@ log () {
 for impl in ENT_GETENTROPY_UNISTD \
             ENT_GETENTROPY_SYSRANDOM \
             ENT_GETRANDOM_SYSCALL \
+            ENT_GETENTROPY_BCRYPTGENRANDOM \
             ENT_DEV_URANDOM
 do
     export IMPL=$impl

pkg/ent/ent.c

@@ -57,6 +57,16 @@ int ent_getentropy(void* buf, size_t len) {
     return 0;
 }
 
+
+// Use `BCryptGenRandom` on Windows ////////////////////////////////////////////
+
+#elif defined(ENT_GETENTROPY_BCRYPTGENRANDOM)
+#include <windows.h>
+#include <bcrypt.h>
+int ent_getentropy(void* buf, size_t len) {
+  return BCryptGenRandom(NULL, (PUCHAR)buf, len, BCRYPT_USE_SYSTEM_PREFERRED_RNG);
+}
+
 #else
-#error "One of these must be set: ENT_DEV_URANDOM, ENT_GETENTROPY_UNISTD, ENT_GETENTROPY_SYSRANDOM, ENT_GETRANDOM_SYSCALL"
+#error "One of these must be set: ENT_GETENTROPY_BCRYPTGENRANDOM, ENT_DEV_URANDOM, ENT_GETENTROPY_UNISTD, ENT_GETENTROPY_SYSRANDOM, ENT_GETRANDOM_SYSCALL"
 #endif

pkg/urbit/.gitattributes

@@ -3,3 +3,4 @@
 tests export-ignore
 hashtable_tests export-ignore
 shell.nix export-ignore
+*.patch -text

pkg/urbit/.gitignore

@@ -1,3 +1,5 @@
 # Configuration Result
 /config.mk
 /include/config.h
+/include/ca-bundle.h
+/include/ivory.h

pkg/urbit/Makefile

@@ -1,4 +1,5 @@
 include config.mk
+include $(foreach dir,$(compat),$(wildcard compat/$(dir)/*.mk))
 
 jets = jets/tree.c $(wildcard jets/*/*.c)
 noun = $(wildcard noun/*.c)
@@ -9,7 +10,9 @@ worker = $(wildcard worker/*.c)
 tests  = $(wildcard tests/*.c)
 bench  = $(wildcard bench/*.c)
 
-common  = $(jets) $(noun) $(ur) $(vere)
+compat := $(foreach dir,$(compat),$(wildcard compat/$(dir)/*.c))
+
+common  = $(jets) $(noun) $(ur) $(vere) $(compat)
 headers = $(shell find include -type f)
 
 common_objs = $(shell echo $(common) | sed 's/\.c/.o/g')
@@ -29,7 +32,7 @@ CFLAGS := $(CFLAGS)
 
 ################################################################################
 
-.PHONY: all test clean mkproper
+.PHONY: all test clean mrproper
 
 ################################################################################
 
@@ -74,9 +77,11 @@ build/urbit-worker: $(common_objs) $(worker_objs)
 	@mkdir -p ./build
 	@$(CC) $^ $(LDFLAGS) -o $@
 
-%.o: %.c $(headers)
+# CCDEPS and CCEXTRA are empty except in MingW build,
+# which uses them to inject a C source transform step
+%.o: %.c $(headers) $(CCDEPS)
 	@echo CC $<
-	@$(CC) -I./include $(CFLAGS) -c $< -o $@
+	@$(CC) -I./include $(CFLAGS) $< $(CCEXTRA) -c -o $@
 
 tags: $(all_srcs) $(headers)
 	ctags $^

pkg/urbit/compat/create-include-files.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# support running off a tarball that doesn't contain binary pills
+(( $($1 ../../bin/ivory.pill) > 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill
+
+poor_mans_xxd () {
+  cch=0
+  echo "unsigned char $2[] = {"
+  while IFS='' read line
+  do
+    for i in $line
+    do
+      echo -n " 0x$i,"
+      cch=$((cch+1))
+    done
+    echo
+  done < <(od -An -v -tx1 $1)
+  echo "};"
+  echo "unsigned int $2_len = $cch;"
+}
+
+[ -e include/ca-bundle.h ] || poor_mans_xxd $2 include_ca_bundle_crt >include/ca-bundle.h
+[ -e include/ivory.h     ] || poor_mans_xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h

pkg/urbit/compat/m1brew/compat.mk

@@ -0,0 +1,14 @@
+# paths to brew packages
+CFLAGS  := $(CFLAGS)  -I/opt/homebrew/include
+LDFLAGS := $(LDFLAGS) -L/opt/homebrew/lib
+# force linker to use static libraries
+LDFLAGS := $(shell compat/m1brew/use-static-libs.sh $(LDFLAGS))
+# add extra osx libraries
+LDFLAGS := $(LDFLAGS) -framework SystemConfiguration
+
+ifdef debug
+CFLAGS  := $(CFLAGS)  -O0 -g
+else
+# clang hangs on noun/allocate.c if -g is specified with -O3
+CFLAGS  := $(CFLAGS)  -O3
+endif

pkg/urbit/compat/m1brew/ed25519.patch

@@ -0,0 +1,13 @@
+diff --git a/Makefile b/Makefile
+new file mode 100644
+--- /dev/null
++++ b/Makefile
+@@ -0,0 +1,8 @@
++.PHONY: all clean
++
++all: *.c *.h
++	$(CC) -c -O3 -Wall -Werror *.c
++	$(AR) rcs libed25519.a *.o
++
++clean:
++	rm -f *.o *.a

pkg/urbit/compat/m1brew/ent.patch

@@ -0,0 +1,12 @@
+diff --git a/configure b/configure
+--- a/configure
++++ b/configure
+@@ -15,7 +15,7 @@ do
+     log "Trying IMPL=$IMPL"
+
+     if IMPL=$impl make >/dev/null 2>/dev/null
+-    then sed -i 's|$(error IMPL must be set)|IMPL='"$impl"'|' Makefile
++    then sed -i "" 's|$(error IMPL must be set)|IMPL='"$impl"'|' Makefile
+          log "IMPL=$IMPL works"
+          exit 0
+     else log "IMPL=$IMPL failed"

pkg/urbit/compat/m1brew/libscrypt.patch

@@ -0,0 +1,17 @@
+diff --git a/Makefile b/Makefile
+index 783c537..3156ee2 100644
+--- a/Makefile
++++ b/Makefile
+@@ -12,10 +12,9 @@ LDFLAGS_EXTRA?=-Wl,-z,relro
+
+ all: reference
+
+-OBJS= crypto_scrypt-nosse.o sha256.o crypto-mcf.o b64.o crypto-scrypt-saltgen.o crypto_scrypt-check.o crypto_scrypt-hash.o slowequals.o
++OBJS= crypto_scrypt-nosse.o sha256.o crypto-mcf.o b64.o slowequals.o
+
+-libscrypt.so.0: $(OBJS) 
+-	$(CC) $(LDFLAGS) -shared -o libscrypt.so.0  $(OBJS) -lm -lc
++libscrypt.a: $(OBJS) 
+ 	ar rcs libscrypt.a  $(OBJS)
+
+ reference: libscrypt.so.0 main.o crypto_scrypt-hexconvert.o

pkg/urbit/compat/m1brew/murmur3.patch

@@ -0,0 +1,13 @@
+diff --git a/makefile b/makefile
+--- a/makefile
++++ b/makefile
+@@ -12,5 +12,9 @@ shared: murmur3.c murmur3.h
+ 	$(CC) -fPIC -O3 -c murmur3.c
+ 	$(CC) -shared -Wl,--export-dynamic murmur3.o -o libmurmur3.so
+
++static: murmur3.c murmur3.h
++	$(CC) -fPIC -O3 -c murmur3.c
++	$(AR) rcs libmurmur3.a murmur3.o
++
+ clean:
+ 	rm -rf example *.o *.so