Uncomplicate ipxe build procedure (#117)

## Description

This PR gets rid of the multiple makefiles / recursive makes approach used to build iPXE. Simplifying the iPXE build description quite a bit.

## Why is this needed

Updating iPXE build config is quite convoluted and makes contributions harder than necessary. As @geekgonecrazy experienced in #79. This is the first steps to possibly addressing #116. 


## How Has This Been Tested?

`make`


## How are existing users impacted? What migration steps/scripts do we need?

Should be easier to mess with iPXE config/builds.

.github/workflows/ci.yaml

@@ -9,6 +9,9 @@ jobs:
     env:
       CGO_ENABLED: 0
     steps:
+      - name: Setup Dynamic Env
+        run: |
+          echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV
       - name: Checkout code
         uses: actions/checkout@v2
         with:
@@ -41,6 +44,8 @@ jobs:
         run: go test -coverprofile=coverage.txt ./...
       - name: upload codecov
         run: bash <(curl -s https://codecov.io/bash)
+      - name: compile iPXE binaries
+        run: nix-shell --run 'make ipxe/bin/ipxe.efi ipxe/bin/snp-hua.efi ipxe/bin/snp-nolacp.efi ipxe/bin/undionly.kpxe'
       - name: compile binaries
         run: nix-shell --run 'make crosscompile'
       - name: Docker Image Tag for Sha

Makefile

@@ -1,30 +1,65 @@
+# Only use the recipes defined in these makefiles
 MAKEFLAGS += --no-builtin-rules
-.PHONY: ${binary} dc gen test
 .SUFFIXES:
+# Delete target files if there's an error
+# This avoids a failure to then skip building on next run if the output is created by shell redirection for example
+# Not really necessary for now, but just good to have already if it becomes necessary later.
+.DELETE_ON_ERROR:
+# Treat the whole recipe as a one shell script/invocation instead of one-per-line 
+.ONESHELL:
+# Use bash instead of plain sh 
+SHELL := bash
+.SHELLFLAGS := -o pipefail -euc
 
 binary := boots
+.PHONY: all ${binary} crosscompile dc gen run test
 all: ${binary}
 
-crosscompile: $(shell git ls-files | grep -v -e vendor -e '_test.go' | grep '.go$$' )
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -v -o ./boots-linux-x86_64 -ldflags="-X main.GitRev=$(shell git rev-parse --short HEAD)"
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./boots-linux-amd64 -ldflags="-X main.GitRev=$(shell git rev-parse --short HEAD)"
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -v -o ./boots-linux-aarch64 -ldflags="-X main.GitRev=$(shell git rev-parse --short HEAD)"
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -v -o ./boots-linux-armv7l -ldflags="-X main.GitRev=$(shell git rev-parse --short HEAD)"
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -v -o ./boots-linux-arm64 -ldflags="-X main.GitRev=$(shell git rev-parse --short HEAD)"
-
+GitRev := $(shell git rev-parse --short HEAD)
+crosscompile: ${binary}
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -v -o ./boots-linux-x86_64 -ldflags="-X main.GitRev=${GitRev}"
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./boots-linux-amd64 -ldflags="-X main.GitRev=${GitRev}"
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -v -o ./boots-linux-aarch64 -ldflags="-X main.GitRev=${GitRev}"
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -v -o ./boots-linux-armv7l -ldflags="-X main.GitRev=${GitRev}"
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -v -o ./boots-linux-arm64 -ldflags="-X main.GitRev=${GitRev}"
 
 # this is quick and its really only for rebuilding when dev'ing, I wish go would
 # output deps in make syntax like gcc does... oh well this is good enough
-${binary}: $(shell git ls-files | grep -v -e vendor -e '_test.go' | grep '.go$$' )
-	CGO_ENABLED=0 go build -v -ldflags="-X main.GitRev=$(shell git rev-parse --short HEAD)"
+${binary}: $(shell git ls-files | grep -v -e vendor -e '_test.go' | grep '.go$$' ) ipxe/bindata.go
+	CGO_ENABLED=0 go build -v -ldflags="-X main.GitRev=${GitRev}"
 
 ifeq ($(origin GOBIN), undefined)
 GOBIN := ${PWD}/bin
 export GOBIN
 endif
+ipxe/bindata.go: ipxe/bin/ipxe.efi ipxe/bin/snp-hua.efi ipxe/bin/snp-nolacp.efi ipxe/bin/undionly.kpxe
+	go-bindata -pkg ipxe -prefix ipxe -o $@ $^
+	gofmt -w $@
+
+include ipxev.mk
+ipxeconfigs := $(wildcard ipxe/ipxe/*.h)
+
+ipxe/bin/ipxe.efi: ipxe/ipxe/build/bin-x86_64-efi/ipxe.efi
+ipxe/bin/snp-nolacp.efi: ipxe/ipxe/build/bin-arm64-efi/snp.efi
+ipxe/bin/undionly.kpxe: ipxe/ipxe/build/bin/undionly.kpxe
+ipxe/bin/ipxe.efi ipxe/bin/snp-nolacp.efi ipxe/bin/undionly.kpxe:
+	cp $^ $@
+
+ipxe/ipxe/build/${ipxev}.tar.gz: ipxev.mk
+	mkdir -p $(@D)
+	curl -fL https://github.com/ipxe/ipxe/archive/${ipxev}.tar.gz > $@
+	echo "${ipxeh}  $@" | sha512sum -c
 
-ipxe/bindata.go:
-	$(MAKE) -C ipxe
+# given  t=$(patsubst ipxe/ipxe/build/%,%,$@)
+# and   $@=ipxe/ipxe/build/*/*
+# t       =                */*
+ipxe/ipxe/build/bin-arm64-efi/snp.efi ipxe/ipxe/build/bin-x86_64-efi/ipxe.efi ipxe/ipxe/build/bin/undionly.kpxe: ipxe/ipxe/build/${ipxev}.tar.gz ipxe/ipxe/build.sh ${ipxeconfigs}
+	+t=$(patsubst ipxe/ipxe/build/%,%,$@)
+	rm -rf $(@D)
+	mkdir -p $(@D)
+	tar -xzf $< -C $(@D)
+	cp ${ipxeconfigs} $(@D)
+	cd $(@D) && ../../build.sh $$t ${ipxev}
 
 ifeq ($(CI),drone)
 run: ${binary}

ipxe/ipxe/.gitignore

@@ -0,0 +1 @@
+build

ipxe/ipxe/build.sh

@@ -1,29 +1,39 @@
 #!/usr/bin/env bash
 
-set -e
+set -eu
 
 # Deps on ubuntu
 # apt-get -y --no-install-recommends install build-essential gcc-aarch64-linux-gnu git liblzma-dev
 
-name=$(basename "$1")
-topdir=${1%%/*}
+build=$1
 version=$2
 short_version="$(echo "$version" | cut -c1-5)"
+
+topdir="ipxe-$version"
+cp ./*.h "$topdir/src/config/local"
 sed -i '/#define OCSP_CHECK/ d' "$topdir/src/config/crypto.h"
-case $name in
-undionly.kpxe)
+
+set -x
+case $build in
+bin/undionly.kpxe)
+	rm "$topdir/src/config/local/isa.h"
 	cp "$topdir/src/config/local/general.undionly.h" "$topdir/src/config/local/general.h"
-	make -C "$topdir/src" VERSION_PATCH=255 EXTRAVERSION="+ ($short_version)" bin/undionly.kpxe
 	;;
-ipxe.efi)
+bin-x86_64-efi/ipxe.efi)
 	cp "$topdir/src/config/local/general.efi.h" "$topdir/src/config/local/general.h"
-	make -C "$topdir/src" VERSION_PATCH=255 EXTRAVERSION="+ ($short_version)" bin-x86_64-efi/ipxe.efi
 	;;
-snp.efi)
-	cp "$topdir/src/config/local/general.aarch64-snp-nolacp.h" "$topdir/src/config/local/general.h"
+bin-arm64-efi/snp.efi)
+	rm "$topdir/src/config/local/isa.h"
+	cp "$topdir/src/config/local/general.efi.h" "$topdir/src/config/local/general.h"
 	# http://lists.ipxe.org/pipermail/ipxe-devel/2018-August/006254.html
 	sed -i '/^WORKAROUND_CFLAGS/ s|^|#|' "$topdir/src/arch/arm64/Makefile"
-	CROSS_COMPILE=aarch64-unknown-linux-gnu- make -C "$topdir/src" VERSION_PATCH=255 EXTRAVERSION="+ ($short_version)" bin-arm64-efi/snp.efi
+	if [[ -z ${CROSS_COMPILE:-} ]]; then
+		export CROSS_COMPILE=aarch64-unknown-linux-gnu-
+	fi
 	;;
-*) echo "unknown target: $1" && exit 1 ;;
+*) echo "unknown target: $1" >&2 && exit 1 ;;
 esac
+
+rm "$topdir"/src/config/local/general.*.h
+make -C "$topdir/src" VERSION_PATCH=255 EXTRAVERSION="+ ($short_version)" "$build"
+cp "$topdir/src/$build" .

ipxe/ipxe/common.h

@@ -0,0 +1,37 @@
+#define DIGEST_CMD            /* Image crypto digest commands */
+#define DOWNLOAD_PROTO_HTTPS  /* Secure Hypertext Transfer Protocol */
+#define IMAGE_TRUST_CMD       /* Image trust management commands */
+#define NET_PROTO_IPV6        /* IPv6 protocol */
+#define NSLOOKUP_CMD          /* DNS resolving command */
+#define NTP_CMD               /* NTP commands */
+#define PARAM_CMD             /* params and param commands, for POSTing to tink */
+#define REBOOT_CMD            /* Reboot command */
+#define VLAN_CMD              /* VLAN commands */
+
+#undef CRYPTO_80211_WEP       /* WEP encryption (deprecated and insecure!) */
+#undef CRYPTO_80211_WPA2      /* Add support for stronger WPA cryptography */
+#undef CRYPTO_80211_WPA       /* WPA Personal, authenticating with passphrase */
+#undef FCMGMT_CMD             /* Fibre Channel management commands */
+#undef IBMGMT_CMD             /* Infiniband management commands */
+#undef IMAGE_PNG              /* PNG image support */
+#undef IMAGE_PNM              /* PNM image support */
+#undef IWMGMT_CMD             /* Wireless interface management commands */
+#undef NET_PROTO_LACP         /* Link Aggregation control protocol */
+#undef NET_PROTO_STP          /* Spanning Tree protocol */
+#undef NVO_CMD                /* Non-volatile option storage commands */
+#undef ROUTE_CMD              /* Routing table management commands */
+#undef VNIC_IPOIB             /* Infiniband IPoIB virtual NICs */
+
+//defined in config/defaults/{efi,pcbios}.h and we don't want
+#undef SANBOOT_PROTO_AOE      /* AoE protocol */
+#undef SANBOOT_PROTO_FCP      /* Fibre Channel protocol */
+#undef SANBOOT_PROTO_HTTP     /* HTTP SAN protocol */
+#undef SANBOOT_PROTO_IB_SRP   /* Infiniband SCSI RDMA protocol */
+#undef SANBOOT_PROTO_ISCSI    /* iSCSI protocol */
+#undef USB_EFI                /* Provide EFI_USB_IO_PROTOCOL interface */
+#undef USB_HCD_EHCI           /* EHCI USB host controller */
+#undef USB_HCD_UHCI           /* UHCI USB host controller */
+#undef USB_HCD_XHCI           /* xHCI USB host controller */
+#undef USB_KEYBOARD           /* USB keyboards */
+
+#define MAX_MODULES 16

ipxe/ipxe/console.h

@@ -1,4 +1 @@
 #define	CONSOLE_SYSLOG
-
-//#define	LOG_LEVEL	LOG_ALL
-

ipxe/ipxe/general.efi.h

@@ -1,18 +1 @@
-#define DIGEST_CMD            /* Image crypto digest commands */
-#define DOWNLOAD_PROTO_HTTPS  /* Secure Hypertext Transfer Protocol */
-#undef IMAGE_COMBOOT         /* COMBOOT */
-#define IMAGE_TRUST_CMD	      /* Image trust management commands */
-#define NET_PROTO_IPV6        /* IPv6 protocol */
-#define NSLOOKUP_CMD          /* DNS resolving command */
-#define NTP_CMD               /* NTP commands */
-#define PARAM_CMD             /* params and param commands, for POSTing to tink */
-#define REBOOT_CMD            /* Reboot command */
-#define VLAN_CMD              /* VLAN commands */
-
-//#undef NET_PROTO_STP
-//#undef NET_PROTO_LACP
-
-//#define BANNER_TIMEOUT          1 // 20
-//#define ROM_BANNER_TIMEOUT      ( 2 * BANNER_TIMEOUT )
-
-#define MAX_MODULES 16
+#include "common.h"

ipxe/ipxe/general.undionly.h

@@ -1,16 +1,3 @@
-#define DIGEST_CMD            /* Image crypto digest commands */
-#define DOWNLOAD_PROTO_HTTPS  /* Secure Hypertext Transfer Protocol */
-#define IMAGE_COMBOOT         /* COMBOOT */
-#define IMAGE_TRUST_CMD	      /* Image trust management commands */
-#define NET_PROTO_IPV6        /* IPv6 protocol */
-#define NSLOOKUP_CMD          /* DNS resolving command */
-#define NTP_CMD               /* NTP commands */
-#define PARAM_CMD             /* params and param commands, for POSTing to tink */
-#define REBOOT_CMD            /* Reboot command */
-#define VLAN_CMD              /* VLAN commands */
+#include "common.h"
 
-//#define BANNER_TIMEOUT		1 // 20
-//#define ROM_BANNER_TIMEOUT	( 2 * BANNER_TIMEOUT )
-
-#define DHCP_USER_CLASS_STRING	DHCP_STRING ( 'i', 'P', 'X', 'E' )
-#define MAX_MODULES 16
+#define IMAGE_COMBOOT          /* COMBOOT */

ipxe/ipxe/isa.h

@@ -0,0 +1 @@
+#define ISA_PROBE_ONLY

ipxev.mk

@@ -0,0 +1,2 @@
+ipxev := a5fb41873dc1dcce5dcc817bf53a0649c01f8cac
+ipxeh := c2e04580877b09ecad45e9e3ec6015d6c770f3c34b953926f3319254e640e821b6fa737f7b8abe383efa81b2cd6651816892b2d12cffc612bafecbd4a23c84f6