Portable mac builds (#2463)

* Portable mac builds

* Correct snarky

* Temporarily try builds on older macs

* Revert "Temporarily try builds on older macs"

This reverts commit 2892fb1.

* Requested changes

* Requested changes

* Snarky back to master

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
 
     build-wallet:
         macos:
-            xcode: "10.1.0"
+            xcode: "10.2.0"
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive

.circleci/config.yml.jinja

@@ -14,7 +14,7 @@ jobs:
 
     build-wallet:
         macos:
-            xcode: "10.1.0"
+            xcode: "10.2.0"
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive

Makefile

@@ -72,6 +72,13 @@ build: git_hooks reformat-diff
 
 dev: codabuilder containerstart build
 
+macos-portable:
+	@rm -rf _build/coda-daemon-macos/
+	@rm -rf _build/coda-daemon-macos.zip
+	@./scripts/macos-portable.sh src/_build/default/app/cli/src/coda.exe src/app/kademlia-haskell/result/bin/kademlia _build/coda-daemon-macos
+	@zip -r _build/coda-daemon-macos.zip _build/coda-daemon-macos/
+	@echo Find coda-daemon-macos.zip inside _build/
+
 ########################################
 ## Lint
 

scripts/debug-dist-coda.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#
+# ONLY USED FOR DEBUGGING
+# Use this script to verify that dist builds build on other machines
+#
+
+# Change to whatever dist.zip you stick on s3
+DIST_ZIP=""
+BUCKET=""
+
+curl "https://s3-us-west-2.amazonaws.com/$BUCKET/$DIST_ZIP" > "$DIST_ZIP"
+
+unzip "$DIST_ZIP" -d dist
+
+cd dist
+
+system_profiler SPSoftwareDataType
+sysctl -n machdep.cpu.brand_string
+
+echo "========="
+echo "== Verify static initilization works"
+echo "========="
+CODA_KADEMLIA_PATH=$PWD/kademlia ./coda.exe -help
+
+# Re-enable if you want to debug illegal instructions
+# lldb -o run ./coda.exe transaction-snark-profiler
+
+echo "========="
+echo "== Verify snarks work"
+echo "========="
+./coda.exe transaction-snark-profiler
+
+rm -rf ~/.coda-config
+
+echo "========="
+echo "== Verify full test"
+echo "========="
+CODA_KADEMLIA_PATH=$PWD/kademlia ./coda.exe integration-tests full-test
+

scripts/macos-portable.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Create a portable Coda build by rewriting the dynamic library loading to look
+# in the current working directory.
+
+set -eou pipefail
+
+if [[ $# -ne 3 ]]; then
+  echo "Usage: $0 <path-to-coda.exe> <path-to-kademlia-binary> <outdir>"
+  exit 1
+fi
+
+LOCAL_CODA_EXE="$(basename "$1")"
+LOCAL_KADEMLIA="$(basename "$2")"
+DIST_DIR="$3"
+
+mkdir -p "$DIST_DIR"
+
+cp "$1" "$DIST_DIR/$LOCAL_CODA_EXE"
+cp "$2" "$DIST_DIR/$LOCAL_KADEMLIA"
+chmod +w "$DIST_DIR/$LOCAL_KADEMLIA"
+
+pushd "$DIST_DIR"
+
+# Set of libraries and binaries we've already rewritten the tables
+SEEN=("")
+
+# `containsElement e array` returns 0 iff e is in the array
+# from https://stackoverflow.com/questions/3685970/check-if-a-bash-array-contains-a-value
+containsElement () {
+  local e match="$1"
+  shift
+  for e; do [[ "$e" == "$match" ]] && return 0; done
+  return 1
+}
+
+# Rewrite the libraries' and binaries' dylib names
+fixup() {
+  local BIN="$1"
+
+  local LIBS=$(otool -l "$BIN" | grep -E '\s+name' | grep -E '/local' | awk '{print $2}')
+
+  echo "$LIBS" | while read lib; do
+    local LOCAL_LIB="$(basename $lib)"
+    if ! containsElement "$LOCAL_LIB" "${SEEN[@]}"; then
+      echo "Moving and rewriting $lib"
+      cp -n "$lib" "$LOCAL_LIB" # no clobber in case we've already moved this lib
+      chmod +w "$LOCAL_LIB"
+      install_name_tool -change "$lib" "@executable_path/$(basename $lib)" "$BIN" || exit 1
+      # Add to our seen set, by adding to the array and then filtering dupes
+      SEEN+=("$BIN")
+      SEEN=($(for v in "${SEEN[@]}"; do echo "$v";done | sort | uniq | xargs))
+      # Recursively call for this lib
+      fixup "$LOCAL_LIB"
+    fi
+  done
+}
+
+# Start with coda.exe
+fixup "$LOCAL_CODA_EXE"
+
+# Fixup kademlia
+K_LIBS=$(otool -l kademlia | grep -E '\s+name' | grep '/nix' | grep -v '\-osx\-' | awk '{print $2}')
+echo "$K_LIBS" | while read lib; do
+  # we already have all the libs from coda.exe thankfully
+  install_name_tool -change "$lib" "@executable_path/$(basename $lib)" "$LOCAL_KADEMLIA"
+done
+

src/external/ocaml-rocksdb/dune

@@ -38,7 +38,18 @@
       'Darwin') jobs=2 ;;
       *) echo I do not know how to build on $(uname) ; exit 1
       esac
-      env ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_GFLAGS=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 ROCKSDB_DISABLE_NUMA=1 ROCKSDB_DISABLE_TBB=1 ROCKSDB_DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALLOC=1 ROCKSDB_DISABLE_BACKTRACE=1 make static_lib -j${jobs}
+      env ROCKSDB_DISABLE_SNAPPY=1 \\
+        ROCKSDB_DISABLE_GFLAGS=1 \\
+        ROCKSDB_DISABLE_LZ4=1 \\
+        ROCKSDB_DISABLE_ZSTD=1 \\
+        ROCKSDB_DISABLE_NUMA=1 \\
+        ROCKSDB_DISABLE_TBB=1 \\
+        ROCKSDB_DISABLE_JEMALLOC=1 \\
+        ROCKSDB_DISABLE_TCMALLOC=1 \\
+        ROCKSDB_DISABLE_BACKTRACE=1 \\
+        PORTABLE=1 \\
+        FORCE_SSE42=1 \\
+        make static_lib -j${jobs}
 
       strip -S librocksdb.a
       popd