mconfig: move network chunk before runtime to install

setuid binary at the end of make install step. gengodep: get all Go dependencies outside of the source tree and exlude Go standard packages. makefile: cli/runtime dependencies are generated and stored in dedicated variable list. Dependency files are updated if go.mod/go.sum changed or if any source file is updated. And add .NOTPARALLEL target to force make to not run in parallel.
sylabs · Dec 20, 2019 · 48e7fab · 48e7fab
1 parent ba94f0c
commit 48e7fab
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 27 deletions.
diff --git a/makeit/gengodep b/makeit/gengodep
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-if [ "$1" != "-v2" ] ; then
+if [ "$1" != "-v3" ] ; then
 	cat 1>&2 <<-EOT
 
 ========================================================================
@@ -18,10 +18,11 @@ fi
 shift
 
 go=$1
-srcdir=$2
+var=$2
 gotags=$3
+depfile=$4
 
-shift 3
+shift 4
 
 # this is needed so that the value we are getting from the makefile does
 # get propagated down to go list.
@@ -33,11 +34,10 @@ else
 	mod_mode=readonly
 fi
 
-"${go}" list -mod=${mod_mode} \
-	-deps \
-	-f '{{ with $d := . }}{{ range $d.GoFiles }}{{ $d.Dir }}/{{ . }} {{ end }}{{ range $d.CgoFiles }}{{ $d.Dir }}/{{ . }} {{ end }}{{ end }}' \
-	-tags "${gotags}" \
-	"$@" |
-tr ' ' '\n' |
-sort -u |
-grep "^${srcdir}"
+template='{{ with $d := . }}{{ if not $d.Standard }}{{ range $d.GoFiles }}{{ printf "%s/%s\n" $d.Dir . }}{{ end }}{{ range $d.CgoFiles }}{{ printf "%s/%s\n" $d.Dir . }}{{ end }}{{ end }}{{ end }}'
+
+godeps=`${go} list -mod=${mod_mode} -deps -f "${template}" -tags "${gotags}" "$@" | sort -u`
+
+for m in ${godeps}; do
+    echo "$var += $m" >> ${depfile}
+done
diff --git a/mconfig b/mconfig
@@ -811,6 +811,11 @@ cat $makeit_interdir/all.mk >> $makeit_makefile
 drawline $makeit_fragsdir/build_cli.mk
 cat $makeit_fragsdir/build_cli.mk >> $makeit_makefile
 
+if [ "$with_network" = 1 ]; then
+    drawline $makeit_fragsdir/build_network.mk
+    cat $makeit_fragsdir/build_network.mk >> $makeit_makefile
+fi
+
 if [ "$build_runtime" = 1 ]; then
 	drawline $makeit_fragsdir/build_runtime.mk
 	cat $makeit_fragsdir/build_runtime.mk >> $makeit_makefile
@@ -821,11 +826,6 @@ if [ "$with_suid" = 1 ]; then
 	cat $makeit_fragsdir/build_runtime_suid.mk >> $makeit_makefile
 fi
 
-if [ "$with_network" = 1 ]; then
-	drawline $makeit_fragsdir/build_network.mk
-	cat $makeit_fragsdir/build_network.mk >> $makeit_makefile
-fi
-
 drawline $makeit_fragsdir/build_scripts.mk
 cat $makeit_fragsdir/build_scripts.mk >> $makeit_makefile
 

diff --git a/mlocal/frags/Makefile.stub b/mlocal/frags/Makefile.stub
@@ -1,3 +1,8 @@
+# as go build already parallelize things, running make in parallel
+# just slow down the whole process and we also requires to respect
+# a specific order
+.NOTPARALLEL:
+
 all: $(ALL)
 
 .PHONY: man

diff --git a/mlocal/frags/build_cli.mk b/mlocal/frags/build_cli.mk
@@ -8,11 +8,15 @@ $(singularity_build_config): $(BUILDDIR)/config.h $(SOURCEDIR)/scripts/go-genera
 
 CLEANFILES += $(singularity_build_config)
 
-# singularity
-singularity_SOURCE := $(shell $(SOURCEDIR)/makeit/gengodep -v2 "$(GO)" "$(SOURCEDIR)" "$(GO_TAGS)" "$(SOURCEDIR)/cmd/singularity")
+# contain singularity_SOURCE variable list
+singularity_deps := $(BUILDDIR_ABSPATH)/singularity.d
+
+$(singularity_deps): $(GO_MODFILES) $(singularity_SOURCE)
+	@echo " GEN GO DEP" $@
+	$(V)$(SOURCEDIR)/makeit/gengodep -v3 "$(GO)" "singularity_SOURCE" "$(GO_TAGS)" "$@" "$(SOURCEDIR)/cmd/singularity"
 
 singularity := $(BUILDDIR)/singularity
-$(singularity): $(singularity_build_config) $(singularity_SOURCE)
+$(singularity): $(singularity_build_config) $(singularity_deps) $(singularity_SOURCE)
 	@echo " GO" $@; echo "    [+] GO_TAGS" \"$(GO_TAGS)\"
 	$(V)$(GO) build $(GO_MODFLAGS) $(GO_BUILDMODE) -tags "$(GO_TAGS)" $(GO_LDFLAGS) $(GO_GCFLAGS) $(GO_ASMFLAGS) \
 		-o $(BUILDDIR)/singularity $(SOURCEDIR)/cmd/singularity
@@ -78,3 +82,4 @@ $(remote_config_INSTALL): $(remote_config)
 
 INSTALLFILES += $(remote_config_INSTALL)
 
+-include $(singularity_deps)
diff --git a/mlocal/frags/build_runtime.mk b/mlocal/frags/build_runtime.mk
@@ -1,11 +1,15 @@
 # This file contains all of the rules for building the singularity runtime
 #   and installing the necessary config files.
 
-starter_SOURCE := $(shell $(SOURCEDIR)/makeit/gengodep -v2 "$(GO)" "$(SOURCEDIR)" "$(GO_TAGS)" "$(SOURCEDIR)/cmd/starter")
-starter_CSOURCE := $(SOURCEDIR)/cmd/starter/c/starter.c \
-                  $(SOURCEDIR)/cmd/starter/c/capability.c \
-                  $(SOURCEDIR)/cmd/starter/c/message.c \
-                  $(SOURCEDIR)/cmd/starter/c/setns.c
+# contain starter_SOURCE variable list
+starter_deps := $(BUILDDIR_ABSPATH)/starter.d
+
+$(starter_deps): $(GO_MODFILES) $(starter_SOURCE)
+	@echo " GEN GO DEP" $@
+	$(V)$(SOURCEDIR)/makeit/gengodep -v3 "$(GO)" "starter_SOURCE" "$(GO_TAGS)" "$@" "$(SOURCEDIR)/cmd/starter"
+
+starter_CSOURCE := $(wildcard $(SOURCEDIR)/cmd/starter/c/*.c)
+starter_CSOURCE += $(wildcard $(SOURCEDIR)/cmd/starter/c/include/*.h)
 
 $(BUILDDIR)/.clean-starter: $(starter_CSOURCE)
 	@echo " GO clean -cache"
@@ -15,7 +19,7 @@ $(BUILDDIR)/.clean-starter: $(starter_CSOURCE)
 
 # starter
 starter := $(BUILDDIR)/cmd/starter/c/starter
-$(starter): $(BUILDDIR)/.clean-starter $(singularity_build_config) $(starter_SOURCE)
+$(starter): $(BUILDDIR)/.clean-starter $(singularity_build_config) $(starter_deps) $(starter_SOURCE)
 	@echo " GO" $@
 	$(V)$(GO) build $(GO_MODFLAGS) $(GO_BUILDMODE) -tags "$(GO_TAGS)" $(GO_LDFLAGS) $(GO_GCFLAGS) $(GO_ASMFLAGS) \
 		-o $@ $(SOURCEDIR)/cmd/starter/main_linux.go
@@ -134,4 +138,4 @@ $(cgroups_config_INSTALL): $(cgroups_config)
 
 INSTALLFILES += $(cgroups_config_INSTALL)
 
-
+-include $(starter_deps)
diff --git a/mlocal/frags/build_runtime_suid.mk b/mlocal/frags/build_runtime_suid.mk
@@ -5,7 +5,7 @@
 starter_suid := $(BUILDDIR)/cmd/starter/c/starter-suid
 starter_suid_INSTALL := $(DESTDIR)$(LIBEXECDIR)/singularity/bin/starter-suid
 
-$(starter_suid): $(BUILDDIR)/.clean-starter $(singularity_build_config) $(starter_SOURCE)
+$(starter_suid): $(starter)
 	@echo " GO" $@; echo "    [+] GO_TAGS" \"$(GO_TAGS_SUID)\"
 	$(V)$(GO) build $(GO_MODFLAGS) $(GO_BUILDMODE) -tags "$(GO_TAGS_SUID)" $(GO_LDFLAGS) $(GO_GCFLAGS) $(GO_ASMFLAGS) \
 		-o $@ $(SOURCEDIR)/cmd/starter/main_linux.go

diff --git a/mlocal/frags/go_common_opts.mk b/mlocal/frags/go_common_opts.mk
@@ -7,6 +7,7 @@ GO_BUILDMODE := -buildmode=default
 GO_GCFLAGS :=
 GO_ASMFLAGS :=
 GO_MODFLAGS := $(if $(wildcard $(SOURCEDIR)/vendor/modules.txt),-mod=vendor,-mod=readonly)
+GO_MODFILES := $(SOURCEDIR)/go.mod $(SOURCEDIR)/go.sum
 GOFLAGS := $(GO_MODFLAGS) -trimpath
 GOPROXY := https://proxy.golang.org