OSv module support

The idea of the patch is basically described in prevoius post:

https://groups.google.com/d/msg/osv-dev/RL2S3AL9TNE/l4XZJo3-lI0J

Whis this patch, you will be able to install OSv apps into disk image on
"make all" stage.

These apps does not require to exist in OSv repository, you can install
apps which is on any git repository or svn repository, or on local
directory.

You'll need to write a config file to add apps, format of the file is
JSON.

Here's a sample of the file:
{
   "modules":[
      {
	 "name":"osv-mruby",
         "type":"git",
         "path":"https://github.com/syuu1228/osv-mruby.git",
         "branch":"master"
      }
   ]
}

If you add "module" on config file, make all calls script/module.py.

This scripts perform "git clone" to fetch repository to $(out)/module,
and invoke "make module" on each module.

"make module" should outputs bootfs.manifest/usr.manifest on module
directory, the script merge bootfs.manifest.skel/usr.manifest.skel and
module local manifests to single file
$(out)/bootfs.manifest/$(out)/usr.manifest.

Here's app Makefile example:

  https://github.com/syuu1228/osv-mruby/blob/master/Makefile

It have "module" target, and the target builds all binaries and
generates *.manifest.

Signed-off-by: Takuya ASADA <syuu@dokukino.com>
Signed-off-by: Pekka Enberg <penberg@cloudius-systems.com>

Makefile

@@ -3,6 +3,7 @@ mode=release
 
 out = build/$(mode)
 submake = $(out)/Makefile
+modulemk = $(out)/module/module.mk
 
 quiet = $(if $V, $1, @echo " $2"; $1)
 silentant = $(if $V,, scripts/silentant.py)
@@ -18,7 +19,7 @@ mgmt = 1
 # idea whether the target has changed or not.  So we call ant from here,
 # and then the main makefile can treat the build products (jars) as inputs
 
-all: $(submake)
+all: $(submake) $(modulemk)
 	$(call quiet, $(silentant) ant -Dmode=$(mode) -Dout=$(abspath $(out)/tests/bench) \
 		-e -f tests/bench/build.xml $(if $V,,-q), ANT tests/bench)
 	$(call only-if, $(mgmt), cd mgmt && ./gradlew --daemon :web:jar build)
@@ -32,6 +33,14 @@ $(submake): Makefile
 	echo 'VPATH = ../..' >> $@
 	echo 'include ../../build.mk' >> $@
 
+$(modulemk): Makefile
+	mkdir -p $(dir $@)
+	echo 'mode = $(mode)' > $@
+	echo 'src = ../../../..' >> $@
+	echo 'out = $(abspath $(out))' >> $@
+	echo 'VPATH = ../../../../' >> $@
+	echo 'include ../../../../build.mk' >> $@
+
 clean:
 	$(call quiet, rm -rf build/$(mode), CLEAN)
 	$(call only-if, $(mgmt), $(call quiet, cd mgmt && ./gradlew --daemon clean >> /dev/null, GRADLE CLEAN))

bootfs.manifest → bootfs.manifest.skel

@@ -76,4 +76,3 @@
 /libzfs.so: libzfs.so
 /zfs.so: zfs.so
 /tools/mkfs.so: tools/mkfs/mkfs.so
-

build.mk

@@ -595,21 +595,21 @@ usr.raw: loader.img
 	$(call quiet, dd if=loader.img of=$@ conv=notrunc > /dev/null 2>&1)
 	$(call quiet, $(src)/scripts/imgedit.py setpartition $@ 2 $(zfs-start) $(zfs-size), IMGEDIT $@)
 
-usr.img: scripts/mkzfs.py usr.manifest $(jni) usr.raw
+usr.img: scripts/mkzfs.py $(out)/usr.manifest $(jni) usr.raw
 	$(call quiet, echo Creating $@ as $(img_format))
 	$(call quiet, qemu-img convert -f raw -O $(img_format) usr.raw $@)
 	$(call quiet, qemu-img resize $@ +10G > /dev/null 2>&1)
-	$(src)/scripts/mkzfs.py -o $@ -d $@.d -m $(src)/usr.manifest \
+	$(src)/scripts/mkzfs.py -o $@ -d $@.d -m $(out)/usr.manifest \
 		-D jdkbase=$(jdkbase) -D gccbase=$(gccbase) -D \
 		glibcbase=$(glibcbase) -D miscbase=$(miscbase) -s $(zfs-start)
 	$(call quiet, $(src)/scripts/imgedit.py setargs $@ $(cmdline), IMGEDIT $@)
 
 $(jni): INCLUDES += -I /usr/lib/jvm/java/include -I /usr/lib/jvm/java/include/linux/
 
-bootfs.bin: scripts/mkbootfs.py bootfs.manifest $(tests) $(tools) \
+bootfs.bin: scripts/mkbootfs.py $(out)/bootfs.manifest $(tests) $(tools) \
 		tests/testrunner.so java/java.so java/runjava.jar \
 		zpool.so zfs.so
-	$(call quiet, $(src)/scripts/mkbootfs.py -o $@ -d $@.d -m $(src)/bootfs.manifest \
+	$(call quiet, $(src)/scripts/mkbootfs.py -o $@ -d $@.d -m $(out)/bootfs.manifest \
 		-D jdkbase=$(jdkbase) -D gccbase=$(gccbase) -D \
 		glibcbase=$(glibcbase) -D miscbase=$(miscbase), MKBOOTFS $@)
 
@@ -639,6 +639,12 @@ gen/include/osv/version.h: $(src)/scripts/gen-version-header
 
 $(src)/build.mk: $(generated-headers)
 
+$(out)/bootfs.manifest: $(src)/scripts/module.py
+	cd $(out)/module && ../../../scripts/module.py bootfs
+
+$(out)/usr.manifest: $(src)/scripts/module.py
+	cd $(out)/module && ../../../scripts/module.py usr
+
 -include $(shell find -name '*.d')
 
 .DELETE_ON_ERROR:

config.json

@@ -0,0 +1,4 @@
+{
+   "modules":[
+   ]
+}

scripts/module.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python2
+
+import json
+import sys
+import os
+import subprocess
+
+f = open("../../../config.json")
+conf = json.load(f)
+f.close()
+
+mtype = sys.argv[1]
+mskel = open("../../../%s.manifest.skel" % mtype)
+m = open("../%s.manifest" % mtype, "w")
+for l in mskel.readlines():
+	m.write(l)
+mskel.close()
+
+for mod in conf["modules"]:
+	mmod_path = "%s/%s.manifest" % (mod["name"], mtype)
+	if os.access(mod["name"], os.F_OK) == False:
+		if mod["type"] == "git":
+			cmd = "git clone -b %s %s %s" % (mod["branch"], mod["path"], mod["name"])
+		elif mod["type"] == "svn":
+			cmd = "svn co %s %s" % (mod["path"], mod["name"])
+		elif mod["type"] == "dir":
+			cmd = "cp -a %s %s" % (mod["path"], mod["name"])
+		else:
+			raise Exception("%s is unknown type" % mod["type"])
+		print cmd
+		subprocess.call([cmd], shell=True)
+	if os.access(mmod_path, os.F_OK) == False:
+		pwd = os.getcwd()
+		print "cd %s" % mod["name"]
+		os.chdir(mod["name"])
+		cmd = "make module"
+		subprocess.call([cmd], shell=True)
+		print "cd -"
+		os.chdir(pwd)
+	print "append %s to %s.manifest" % (mmod_path, mtype)
+	mmod = open(mmod_path)
+	for l in mmod.readlines():
+		m.write(l)
+	mmod.close()
+m.close()