Add Makefile targets to implement the buildbot steps.

Android/README.rst

@@ -108,7 +108,7 @@ Makefile targets
 ^^^^^^^^^^^^^^^^
 *build*
     Compile the native Python interpreter. Cross-compile the external libraries
-    and Python.
+    and Python. This is the default target.
 
 *dist*
     Make a distribution consisting of:
@@ -155,17 +155,9 @@ Makefile targets
 
         Do you wish to create a custom hardware profile? [no]
 
-    To kill an emulator that cannot be closed using its GUI, follow these steps:
-
-    - Create once and for all the following empty file in the user's home
-      directory::
-
-        $ > ~/.emulator_console_auth_token
-
-    - Start a telnet session on port 5554 and run the kill command::
-
-        $ telnet localhost 5554
-            kill
+*kill_emulator*
+    Kill the emulator. Useful when the emulator refuses to be shutdown from its
+    GUI.
 
 *adb_shell*
     Create an adb_ shell on the emulator.

Android/build.mk

@@ -17,7 +17,7 @@ py_fullname = python$(py_version)$(abiflags)
 export STDLIB_DIR := lib/python$(py_version)
 
 
-# Target names.
+# Target variables names.
 python := $(py_host_dir)/python
 config_status := $(py_host_dir)/config.status
 export PYTHON_ZIP := $(DIST_DIR)/$(py_name)-$(BUILD_TYPE).zip
@@ -37,12 +37,13 @@ $(native_build_dir)/config.status: $(py_srcdir)/configure
 
 native_python: $(native_build_dir)/config.status
 	@echo "---> Build the native interpreter."
-	cd $(native_build_dir); $(MAKE)
+	$(MAKE) -C $(native_build_dir)
 
 # Target-specific exported variables.
 $(config_status):           export CPPFLAGS := -I$(PY_DESTDIR)/$(SYS_EXEC_PREFIX)/include
 $(config_status):           export LDFLAGS := -L$(PY_DESTDIR)/$(SYS_EXEC_PREFIX)/lib
-$(python) python_dist:      export PATH := $(native_build_dir):$(PATH)
+build configure host python_dist: \
+                            export PATH := $(native_build_dir):$(PATH)
 external_libraries:         export CC := $(CC)
 external_libraries openssl: export AR := $(AR)
 external_libraries openssl: export LD := $(LD)
@@ -80,13 +81,24 @@ $(config_status): $(makefile) $(py_srcdir)/configure
 	    $(config_args)
 
 $(python): native_python external_libraries openssl $(config_status)
+	$(MAKE) host
+
+configure: native_python external_libraries openssl
+	@rm -f $(config_status)
+	$(MAKE) $(config_status)
+
+host:
 	@echo "---> Build Python for $(BUILD_TYPE)."
-	cd $(py_host_dir); $(MAKE)
+	@if test -f $(config_status); then \
+	    $(MAKE) -C $(py_host_dir) all; \
+	else \
+	    echo "Error: please run 'make config', missing $(config_status)"; \
+	    false; \
+	fi
 
 python_dist: $(python)
 	@echo "---> Install Python for $(BUILD_TYPE)."
-	cd $(py_host_dir); \
-	    $(MAKE) DESTDIR=$(PY_DESTDIR) install
+	$(MAKE) DESTDIR=$(PY_DESTDIR) -C $(py_host_dir) install
 
 $(PYTHON_ZIP): python_dist
 	@echo "---> Zip the machine-specific Python library."
@@ -140,6 +152,9 @@ distclean:
 	rm -f $(BUILD_DIR)/python
 	rm -rf $(PY_DESTDIR)
 
+hostclean:
+	$(MAKE) -C $(py_host_dir) distclean
+
 # Remove everything for the given ANDROID_API and ANDROID_ARCH except the avd.
 clean: distclean
 	rm -rf $(BUILD_DIR)/python-native
@@ -167,5 +182,5 @@ else
       directory.)
 endif
 
-.PHONY: dist native_python external_libraries openssl python_dist \
-        distclean clean
+.PHONY: build configure host native_python external_libraries openssl \
+        dist python_dist distclean hostclean clean

Android/emulator.mk

@@ -13,24 +13,36 @@ endif
 # Rules.
 emulator: emulator_checks _emulator adb_shell
 
+emulator_install: emulator_checks dist sdclean _emulator
+	@echo "---> Install Python on the avd and start the emulator."
+	$(native_python_exe) $(py_srcdir)/Android/tools/install.py
+
 adb_shell:
 	@echo "---> Run an adb shell."
 	$(native_python_exe) $(py_srcdir)/Android/tools/python_shell.py
 	@$(ADB) shell
 
-install: emulator_checks dist sdclean _emulator
-	@echo "---> Install Python on the avd, start the emulator and run an adb shell."
-	$(native_python_exe) $(py_srcdir)/Android/tools/install.py
+install: emulator_install
+	@echo "---> Run an adb shell."
 	$(native_python_exe) $(py_srcdir)/Android/tools/python_shell.py
 	@$(ADB) shell
 
 ifneq ($(PYTHON_ARGS), )
-python: emulator_checks dist sdclean _emulator
-	@echo "---> Install Python on the avd, start the emulator and run <python $(PYTHON_ARGS)>."
-	$(native_python_exe) $(py_srcdir)/Android/tools/install.py
-	$(native_python_exe) $(py_srcdir)/Android/tools/python_shell.py "$(PYTHON_ARGS)"
+python: emulator_install
+	@echo "---> Run <python $(PYTHON_ARGS)>."
+	-$(native_python_exe) $(py_srcdir)/Android/tools/python_shell.py "$(PYTHON_ARGS)"
+	$(MAKE) kill_emulator
 endif
 
+buildbottest: TESTRUNNER := $(native_python_exe) $(py_srcdir)/Android/tools/python_shell.py
+buildbottest: TESTRUNNER += $(TESTPYTHONOPTS) $(SYS_EXEC_PREFIX)/bin/run_tests.py
+buildbottest: export PY_SRCDIR := $(py_srcdir)
+buildbottest: export PATH := $(native_build_dir):$(PATH)
+buildbottest: emulator_install
+	@echo "---> Run buildbottest."
+	-$(MAKE) -C $(py_host_dir) TESTRUNNER="$(TESTRUNNER)" buildbottest
+	$(MAKE) kill_emulator
+
 gdb: export APP_ABI := $(APP_ABI)
 gdb: export PY_HOST_DIR := $(py_host_dir)
 gdb: export LIB_DYNLOAD = $(py_host_dir)/$(shell cat $(py_host_dir)/pybuilddir.txt)
@@ -53,6 +65,10 @@ $(avd_dir)/sdcard.img:
 	mkdir -p $(avd_dir)
 	$(ANDROID_SDK_ROOT)/tools/mksdcard -l sl4a 512M $(avd_dir)/sdcard.img
 
+kill_emulator:
+	@echo "---> Kill the emulator."
+	$(native_python_exe) $(py_srcdir)/Android/tools/kill_emulator.py
+
 emulator_checks:
 	@echo "---> Check that an emulator is not currently running."
 	@if test -f "$(native_python_exe)"; then \
@@ -81,4 +97,5 @@ avdclean: sdclean
 	fi
 	-rmdir $(avd_dir)
 
-.PHONY: emulator adb_shell install python gdb sdclean avdclean _emulator emulator_checks
+.PHONY: emulator adb_shell install python buildbottest gdb \
+        sdclean avdclean _emulator kill_emulator emulator_checks emulator_install

Android/tools/android_utils.py

@@ -112,9 +112,9 @@ def adb_push_to_dir(src, dest):
 
 def run_script(script_path):
     """Push a script to the emulator and run it."""
-    sys_prefix = os.environ['SYS_PREFIX']
-    adb_push_to_dir(script_path, sys_prefix)
-    script_path = os.path.join(sys_prefix, os.path.basename(script_path))
+    bin_dir = os.path.join(os.environ['SYS_EXEC_PREFIX'], 'bin')
+    adb_push_to_dir(script_path, bin_dir)
+    script_path = os.path.join(bin_dir, os.path.basename(script_path))
     print()
     print('Running %s' % script_path, flush=True)
     subprocess.run([os.environ['ADB'], 'shell', 'sh %s' % script_path])

Android/tools/install.py

@@ -41,6 +41,12 @@ def install():
                    os.environ['ZIPBASE_DIR'],
                    os.environ['ANDROID_APP_DIR'])
 
+    srcdir = os.environ.get('PY_SRCDIR')
+    if srcdir:
+        # Push the script run by buildbottest.
+        bin_dir = join(os.environ['SYS_EXEC_PREFIX'], 'bin')
+        adb_push_to_dir(join(srcdir, 'Tools/scripts/run_tests.py'), bin_dir)
+
 if __name__ == "__main__":
     try:
         install()

Android/tools/kill_emulator.py

@@ -0,0 +1,28 @@
+"""Kill the emulator."""
+
+import sys
+import os
+import telnetlib
+
+def main():
+    try:
+        with telnetlib.Telnet('localhost', 5554) as tn:
+            idx, _, bytes_read = tn.expect([b'Android Console'], timeout=5)
+            if idx != 0:
+                if bytes_read:
+                    print(bytes_read, file=sys.stderr)
+                return 'Timed out, not an Android Console'
+            with open(os.path.expanduser('~/.emulator_console_auth_token'),
+                                         'rb') as f:
+                token = f.read()
+            tn.write(b'auth ' + token + b'\n')
+            tn.write(b'kill\n')
+            print(tn.read_all().decode('ascii'), file=sys.stderr)
+    except OSError as e:
+        return e
+
+if __name__ == "__main__":
+    err = main()
+    if err is not None:
+        print('Error: Cannot telnet to the Android Console: %s.' % err)
+        sys.exit(1)

Android/tools/python_shell.py

@@ -17,11 +17,13 @@
 
 from android_utils import (adb_push_to_dir, run_script, AndroidError)
 
+# Maximum file name size on nearly all current file systems.
+MAX_FNAME_SIZE = 255
 USAGE = """
 ===========================================================================
 
 Set the environment by sourcing python_shell.sh with the following command:
-    '. ${SYS_PREFIX}/python_shell.sh'
+    '. ${SYS_EXEC_PREFIX}/bin/python_shell.sh'
 
 ===========================================================================
 
@@ -30,16 +32,14 @@
 def build_script():
     script = """
         # Set the environment variables and change the current directory to
-        # ANDROID_APP_DIR.
+        # SYS_EXEC_PREFIX.
         export HOME=$ANDROID_APP_DIR
-        sys_exec_prefix=$$HOME/$ZIPBASE_DIR
-        export PATH=$$sys_exec_prefix/bin:$$PATH
-        export LD_LIBRARY_PATH=$$sys_exec_prefix/lib
+        export PATH=$SYS_EXEC_PREFIX/bin:$$PATH
+        export LD_LIBRARY_PATH=$SYS_EXEC_PREFIX/lib
         export TERM=linux
-        export TERMINFO=$$sys_exec_prefix/share/terminfo
-        export INPUTRC=$$sys_exec_prefix/etc/inputrc
-        unset sys_exec_prefix
-        cd $ANDROID_APP_DIR
+        export TERMINFO=$SYS_EXEC_PREFIX/share/terminfo
+        export INPUTRC=$SYS_EXEC_PREFIX/etc/inputrc
+        cd $SYS_EXEC_PREFIX
 
     """
     script = textwrap.dedent(script)
@@ -54,7 +54,12 @@ def build_script():
     else:
         args = ''.join(map(lambda c: c if c.isalnum() else '_',
                            '_'.join(sys.argv[1:])))
-        script_name = 'python_%s.sh' % args
+        s = 'python_%s.sh' % args
+        l = len(s)
+        if l > MAX_FNAME_SIZE:
+            slice = MAX_FNAME_SIZE // 2 - 2
+            s = s[:slice] + '____' + s[l - slice:]
+        script_name = s
         script += 'python %s\n' % ' '.join(sys.argv[1:])
 
     script_path = os.path.join(os.environ['DIST_DIR'], script_name)
@@ -65,7 +70,6 @@ def build_script():
 
 def main():
     script_path = build_script()
-    sys_prefix = os.environ['SYS_PREFIX']
 
     # The adb shell is mksh (The MirBSD Korn Shell) and it would be possible
     # to update its configuration file at /system/etc/mkshrc on Android to add
@@ -75,7 +79,8 @@ def main():
     # See
     # http://stackoverflow.com/questions/11950131/android-adb-shell-ash-or-ksh.
     if len(sys.argv) == 1:
-        adb_push_to_dir(script_path, sys_prefix)
+        bin_dir = os.path.join(os.environ['SYS_EXEC_PREFIX'], 'bin')
+        adb_push_to_dir(script_path, bin_dir)
         print(string.Template(USAGE).substitute(os.environ))
     else:
         run_script(script_path)