enhance distribute.sh to mark recipe as built or not, and allow the r…

…ecipe to also indicate if it should be rebuilt or not + add -u "mod1 mod2" to force update of recipes (can be used when developing on android module, or specific P4A_module_DIR + standardize HOSTPYTHON and SITEPACKAGES_PATH

distribute.sh

@@ -28,6 +28,8 @@ PACKAGES_PATH="$ROOT_PATH/.packages"
 SRC_PATH="$ROOT_PATH/src"
 JNI_PATH="$SRC_PATH/jni"
 DIST_PATH="$ROOT_PATH/dist/default"
+SITEPACKAGES_PATH="$BUILD_PATH/python-install/lib/python2.7/site-packages/"
+HOSTPYTHON="$BUILD_PATH/python-install/bin/python.host"
 
 # Tools
 export LIBLINK_PATH="$BUILD_PATH/objects"
@@ -213,19 +215,18 @@ function pop_arm() {
 
 function usage() {
 	echo "Python for android - distribute.sh"
-	echo "This script create a directory will all the libraries wanted"
 	echo 
-	echo "Usage:   ./distribute.sh [options] directory"
-	echo "Example: ./distribute.sh -m 'pil kivy' dist"
-	echo
-	echo "Options:"
+	echo "Usage:   ./distribute.sh [options]"
 	echo
 	echo "  -d directory           Name of the distribution directory"
 	echo "  -h                     Show this help"
 	echo "  -l                     Show a list of available modules"
 	echo "  -m 'mod1 mod2'         Modules to include"
 	echo "  -f                     Restart from scratch (remove the current build)"
-        echo "  -x                     display expanded values (execute 'set -x')"
+	echo "  -x                     display expanded values (execute 'set -x')"
+	echo
+	echo "For developers:"
+	echo "  -u 'mod1 mod2'         Modules to update (if already compiled)"
 	echo
 	exit 0
 }
@@ -330,6 +331,11 @@ function run_prepare() {
 		try rm -rf $BUILD_PATH
 		try rm -rf $SRC_PATH/obj
 		try rm -rf $SRC_PATH/libs
+		pushd $JNI_PATH
+		push_arm
+		try ndk-build clean
+		pop_arm
+		popd
 	fi
 
 	# create build directory if not found
@@ -585,11 +591,40 @@ function run_prebuild() {
 
 function run_build() {
 	info "Run build"
+
+	modules_update=($MODULES_UPDATE)
+
 	cd $BUILD_PATH
+
 	for module in $MODULES; do
-		fn=$(echo build_$module)
-		debug "Call $fn"
-		$fn
+		fn="build_$module"
+		shouldbuildfn="shouldbuild_$module"
+		MARKER_FN="$BUILD_PATH/.mark-$module"
+
+		# if the module should be updated, then remove the marker.
+		in_array $module "${modules_update[@]}"
+		if [ $? -ne 255 ]; then
+			debug "$module detected to be updated"
+			rm -f "$MARKER_FN"
+		fi
+
+		# if shouldbuild_$module exist, call it to see if the module want to be
+		# built again
+		DO_BUILD=1
+		if [ "$(type -t $shouldbuildfn)" == "function" ]; then
+			$shouldbuildfn
+		fi
+
+		# if the module should be build, or if the marker is not present,
+		# do the build
+		if [ "X$DO_BUILD" == "X1" ] || [ ! -f "$MARKER_FN" ]; then
+			debug "Call $fn"
+			rm -f "$MARKER_FN"
+			$fn
+			touch "$MARKER_FN"
+		else
+			debug "Skipped $fn"
+		fi
 	done
 }
 
@@ -633,7 +668,7 @@ function run_pymodules_install() {
 	try echo "$PYMODULES" | try sed 's/\ /\n/g' > requirements.txt
 
 	debug "Install pure-python modules via pip in venv"
-	try bash -c "source venv/bin/activate && env CC=/bin/false CXX=/bin/false $PIP install --target '$BUILD_PATH/python-install/lib/python2.7/site-packages' --download-cache '$PACKAGES_PATH' -r requirements.txt"
+	try bash -c "source venv/bin/activate && env CC=/bin/false CXX=/bin/false $PIP install --target '$SITEPACKAGES_PATH' --download-cache '$PACKAGES_PATH' -r requirements.txt"
 
 }
 
@@ -656,7 +691,7 @@ function run_distribute() {
 	try cp -a $BUILD_PATH/blacklist.txt .
 
 	debug "Copy python distribution"
-	$BUILD_PATH/python-install/bin/python.host -OO -m compileall $BUILD_PATH/python-install
+	$HOSTPYTHON -OO -m compileall $BUILD_PATH/python-install
 	try cp -a $BUILD_PATH/python-install .
 
 	debug "Copy libs"
@@ -679,19 +714,11 @@ function run_distribute() {
 	try find . | grep -E '*\.(py|pyc|so\.o|so\.a|so\.libs)$' | xargs rm
 
 	# we are sure that all of theses will be never used on android (well...)
-	try rm -rf test
 	try rm -rf ctypes
 	try rm -rf lib2to3
-	try rm -rf lib-tk
 	try rm -rf idlelib
-	try rm -rf unittest/test
-	try rm -rf json/tests
-	try rm -rf distutils/tests
-	try rm -rf email/test
-	try rm -rf bsddb/test
 	try rm -rf config/libpython*.a
 	try rm -rf config/python.o
-	try rm -rf curses
 	try rm -rf lib-dynload/_ctypes_test.so
 	try rm -rf lib-dynload/_testcapi.so
 
@@ -746,7 +773,7 @@ function arm_deduplicate() {
 
 
 # Do the build
-while getopts ":hvlfxm:d:s" opt; do
+while getopts ":hvlfxm:u:d:s" opt; do
 	case $opt in
 		h)
 			usage
@@ -765,6 +792,9 @@ while getopts ":hvlfxm:d:s" opt; do
 		m)
 			MODULES="$OPTARG"
 			;;
+		u)
+			MODULES_UPDATE="$OPTARG"
+			;;
 		d)
 			DIST_PATH="$ROOT_PATH/dist/$OPTARG"
 			;;

docs/source/conf.py

@@ -48,9 +48,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.1'
+version = '1.2'
 # The full version, including alpha/beta/rc tags.
-release = '1.1'
+release = '1.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/usage.rst

@@ -48,6 +48,7 @@ Available options to `distribute.sh`::
     -l                     Show a list of available modules
     -m 'mod1 mod2'         Modules to include
     -f                     Restart from scratch (remove the current build)
+    -u 'mod1 mod2'         Modules to update (if already compiled)
 
 Step 2: package your application
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

recipes/android/recipe.sh

@@ -16,6 +16,12 @@ function prebuild_android() {
 	fi
 }
 
+function shouldbuild_android() {
+	if [ -d "$SITEPACKAGES_PATH/android" ]; then
+		DO_BUILD=0
+	fi
+}
+
 function build_android() {
 	cd $BUILD_android
 
@@ -31,8 +37,8 @@ function build_android() {
 
 	# cythonize
 	try find . -iname '*.pyx' -exec cython {} \;
-	try $BUILD_PATH/python-install/bin/python.host setup.py build_ext -v
-	try $BUILD_PATH/python-install/bin/python.host setup.py install -O2
+	try $HOSTPYTHON setup.py build_ext -v
+	try $HOSTPYTHON setup.py install -O2
 
 	unset LDSHARED
 

recipes/audiostream/recipe.sh

@@ -11,14 +11,15 @@ function prebuild_audiostream() {
 	cd $BUILD_audiostream
 }
 
+function shouldbuild_audiostream() {
+	if [ -d "$SITEPACKAGES_PATH/audiostream" ]; then
+		DO_BUILD=0
+	fi
+}
+
 function build_audiostream() {
 	cd $BUILD_audiostream
 
-	if [ -d "$BUILD_PATH/python-install/lib/python2.7/site-packages/audiostream" ]; then
-		#return
-		true
-	fi
-
 	push_arm
 
 	# build python extension
@@ -27,10 +28,10 @@ function build_audiostream() {
 	export LDFLAGS="$LDFLAGS -lm -L$LIBS_PATH"
 	export AUDIOSTREAM_ROOT="$BUILD_audiostream/build/audiostream/armeabi-v7a"
 	try cd $BUILD_audiostream
-	$BUILD_PATH/python-install/bin/python.host setup.py build_ext &>/dev/null
+	$HOSTPYTHON setup.py build_ext &>/dev/null
 	try find . -iname '*.pyx' -exec cython {} \;
-	try $BUILD_PATH/python-install/bin/python.host setup.py build_ext -v
-	try $BUILD_PATH/python-install/bin/python.host setup.py install -O2
+	try $HOSTPYTHON setup.py build_ext -v
+	try $HOSTPYTHON setup.py install -O2
 	try cp -a audiostream/platform/android/org $JAVACLASS_PATH
 
 	pop_arm

recipes/cymunk/recipe.sh

@@ -11,6 +11,12 @@ function prebuild_cymunk() {
 	true
 }
 
+function shouldbuild_cymunk() {
+	if [ -d "$SITEPACKAGES_PATH/cymunk" ]; then
+		DO_BUILD=0
+	fi
+}
+
 function build_cymunk() {
 	cd $BUILD_cymunk
 
@@ -19,7 +25,7 @@ function build_cymunk() {
 	export LDSHARED="$LIBLINK"
 
 	try find . -iname '*.pyx' -exec cython {} \;
-	try $BUILD_PATH/python-install/bin/python.host setup.py build_ext -v
+	try $HOSTPYTHON setup.py build_ext -v
 	try find build/lib.* -name "*.o" -exec $STRIP {} \;
 
 	export PYTHONPATH=$BUILD_hostpython/Lib/site-packages

recipes/docutils/recipe.sh

@@ -11,12 +11,13 @@ function prebuild_docutils() {
     true
 }
 
-function build_docutils() {
-    if [ -d "$BUILD_PATH/python-install/lib/python2.7/site-packages/docutils" ]; then
-        #return
-        true
-    fi
+function shouldbuild_docutils() {
+	if [ -d "$SITEPACKAGES_PATH/docutils" ]; then
+		DO_BUILD=0
+	fi
+}
 
+function build_docutils() {
     cd $BUILD_docutils
 
     push_arm
@@ -25,10 +26,10 @@ function build_docutils() {
     export LDSHARED="$LIBLINK"
 
     # fake try to be able to cythonize generated files
-    $BUILD_PATH/python-install/bin/python.host setup.py build_ext
+    $HOSTPYTHON setup.py build_ext
     try find . -iname '*.pyx' -exec cython {} \;
-    try $BUILD_PATH/python-install/bin/python.host setup.py build_ext -v
-    try $BUILD_PATH/python-install/bin/python.host setup.py install -O2
+    try $HOSTPYTHON setup.py build_ext -v
+    try $HOSTPYTHON setup.py install -O2
 
     unset LDSHARED
     pop_arm

recipes/ffmpeg/recipe.sh

@@ -16,13 +16,14 @@ function prebuild_ffmpeg() {
 	fi
 }
 
-function build_ffmpeg() {
-	cd $BUILD_ffmpeg
-
-	if [ -d "$BUILD_PATH/python-install/lib/python2.7/site-packages/ffmpeg" ]; then
-		return
+function shouldbuild_ffmpeg() {
+	if [ -d "$SITEPACKAGES_PATH/ffmpeg" ]; then
+		DO_BUILD=0
 	fi
+}
 
+function build_ffmpeg() {
+	cd $BUILD_ffmpeg
 
 	# build ffmpeg
 	export NDK=$ANDROIDNDK
@@ -39,8 +40,8 @@ function build_ffmpeg() {
 	export FFMPEG_ROOT="$BUILD_ffmpeg/build/ffmpeg/armeabi-v7a"
 	try cd $BUILD_ffmpeg/python
 	try find . -iname '*.pyx' -exec cython {} \;
-	try $BUILD_PATH/python-install/bin/python.host setup.py build_ext -v
-	try $BUILD_PATH/python-install/bin/python.host setup.py install -O2
+	try $HOSTPYTHON setup.py build_ext -v
+	try $HOSTPYTHON setup.py install -O2
 
 	pop_arm
 }

recipes/gevent/recipe.sh

@@ -11,18 +11,20 @@ function prebuild_gevent() {
     true
 }
 
+function shouldbuild_gevent() {
+	if [ -d "$SITEPACKAGES_PATH/gevent" ]; then
+		DO_BUILD=0
+	fi
+}
+
 function build_gevent() {
     cd $BUILD_gevent
 
-    if [ -d "$BUILD_PATH/python-install/lib/python2.7/site-packages/gevent" ]; then
-        return
-    fi
-
     push_arm
     export CFLAGS="$CFLAGS -I$BUILD_libevent/build/include"
     export LDFLAGS="$LDFLAGS -L$LIBS_PATH -L$BUILD_libevent/build/lib/"
 
-    try $BUILD_PATH/python-install/bin/python.host setup.py install -O2
+    try $HOSTPYTHON setup.py install -O2
     pop_arm
 }
 

recipes/greenlet/recipe.sh

@@ -12,16 +12,17 @@ function prebuild_greenlet() {
     true
 }
 
+function shouldbuild_greenlet() {
+	if [ -d "$SITEPACKAGES_PATH/greenlet" ]; then
+		DO_BUILD=0
+	fi
+}
+
 function build_greenlet() {
     cd $BUILD_greenlet
 
-    if [ -d "$BUILD_PATH/python-install/lib/python2.7/site-packages/greenlet.so" ]; then
-        return
-    fi
-
     push_arm
-
-    try $BUILD_PATH/python-install/bin/python.host setup.py install -O2
+    try $HOSTPYTHON setup.py install -O2
     pop_arm
 }
 

recipes/hostpython/recipe.sh

@@ -13,14 +13,16 @@ function prebuild_hostpython() {
 	try cp $RECIPE_hostpython/Setup Modules/Setup
 }
 
-function build_hostpython() {
-	# placeholder for building
+function shouldbuild_hostpython() {
 	cd $BUILD_hostpython
-
-	# don't do the build if we already got hostpython binary
 	if [ -f hostpython ]; then
-		return
+		DO_BUILD=0
 	fi
+}
+
+function build_hostpython() {
+	# placeholder for building
+	cd $BUILD_hostpython
 
     try ./configure
     try make -j5