Python3 compatibility

.travis.yml

@@ -7,37 +7,54 @@ matrix:
   include:
     - name: "Ubuntu: no configure flags"
       compiler: gcc
+      env:
+       - PYTHON_VERSION=2.7
+    - name: "Ubuntu: py3.6 distcheck"
+      compiler: gcc
+      env:
+       - DISTCHECK=t
+       - PYTHON_VERSION=3.6
     - name: "Ubuntu: gcc-8 --with-flux-security/caliper, distcheck"
       compiler: gcc-8
       env:
        - CC=gcc-8
        - CXX=g++-8
        - ARGS="--with-flux-security --enable-caliper"
        - DISTCHECK=t
-    - name: "Ubuntu: clang-6.0 chain-lint --with-flux-security"
+       - PYTHON_VERSION=2.7
+    - name: "Ubuntu: clang-6.0 chain-lint --with-flux-security --enable-pylint"
       compiler: clang-6.0
       env:
        - CC=clang-6.0
        - CXX=clang++-6.0
        - chain_lint=t
-       - ARGS="--with-flux-security"
+       - ARGS="--with-flux-security --enable-pylint"
+       - PYTHON_VERSION=2.7
     - name: "Ubuntu: COVERAGE=t, --with-flux-security --enable-caliper"
       compiler: gcc
       env:
        - COVERAGE=t
        - ARGS="--with-flux-security --enable-caliper"
+       - PYTHON_VERSION=2.7
     - name: "Ubuntu: TEST_INSTALL docker-deploy"
       compiler: gcc
       env:
        - ARGS="--with-flux-security --enable-caliper"
        - TEST_INSTALL=t
        - DOCKER_TAG=t
+       - PYTHON_VERSION=2.7
     - name: "Centos 7: --with-flux-security --enable-caliper docker-deploy"
       compiler: gcc
       env:
        - ARGS="--with-flux-security --enable-caliper --prefix=/usr"
        - IMG=centos7-base
        - DOCKER_TAG=t
+       - PYTHON_VERSION=2.7
+    - name: "Centos 7: py3.4"
+      compiler: gcc
+      env:
+       - IMG=centos7-base
+       - PYTHON_VERSION=3.4
 
 env:
   global:
@@ -68,7 +85,6 @@ before_install:
 script:
  - |
   src/test/docker/docker-run-checks.sh -j2 \
-    --quiet \
     --image=${IMG} \
     ${TAGNAME:+--tag=${TAGNAME}} \
     -- ${ARGS}

Makefile.am

@@ -4,6 +4,7 @@ SUBDIRS = . src doc etc t
 
 EXTRA_DIST = \
 	config/tap-driver.sh \
+	config/tap-driver.py \
 	DISCLAIMER.LLNS \
 	README.md
 

config/tap-driver.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import sys
+import os
+from os import path
+
+def main():
+    arguments = sys.argv[1:] # 0 is me
+    try:
+        args_split_point = arguments.index('--')
+        driver_args = arguments[:args_split_point]
+        test_command = arguments[args_split_point+1:]
+    except ValueError:
+        for idx, value in enumerate(arguments):
+            if not value.startswith('--'):
+                driver_args = arguments[:idx]
+                test_command = arguments[idx:]
+                break
+
+    driver = path.join(path.dirname(path.realpath(__file__)), "tap-driver.sh")
+    full_command = [driver] + driver_args + ["--", sys.executable] + test_command
+    os.execv(driver, full_command)
+
+if __name__ == "__main__":
+    main()

configure.ac

@@ -146,14 +146,14 @@ AC_ARG_ENABLE(python,
 AC_MSG_RESULT($enable_python)
 
 if test "$enable_python" = "yes"; then
-  AM_PATH_PYTHON([2.7])
+  AX_PYTHON_DEVEL([>='2.7'])
+  AM_PATH_PYTHON([$ac_python_version])
   if test "X$PYTHON" != "X"; then
     # Flag for PYTHON_LDFLAGS workaround below.
     if test -n "$PYTHON_LDFLAGS"; then
       ac_python_ldflags_set_by_user=true
     fi
 
-    AX_PYTHON_DEVEL
     #TODO: write this manually to add a version check
     AM_CHECK_PYMOD(cffi,
                    [cffi.__version_info__ >= (1,1)],
@@ -179,6 +179,8 @@ if test "$enable_python" = "yes"; then
   fi
   AS_VAR_SET(fluxpydir, $pyexecdir/flux)
   AC_SUBST(fluxpydir)
+  AS_VAR_SET(fluxsodir, $pyexecdir/_flux)
+  AC_SUBST(fluxsodir)
   AS_VAR_SET(fluxpymoddir, $pyexecdir/flux/modules)
   AC_SUBST(fluxpymoddir)
 
@@ -330,6 +332,7 @@ AC_CONFIG_FILES( \
   src/bindings/python/Makefile \
   src/bindings/python/flux/Makefile \
   src/bindings/python/flux/core/Makefile \
+  src/bindings/python/_flux/Makefile \
   src/broker/Makefile \
   src/cmd/Makefile \
   src/connectors/Makefile \
@@ -367,11 +370,6 @@ AC_CONFIG_FILES( \
   t/fluxometer/conf.lua.installed \
 )
 
-AC_CONFIG_FILES(
-  [t/t9990-python-tests.t:t/t9990-python-tests.t.in],
-  [chmod +x t/t9990-python-tests.t]
-                )
-
 AC_CONFIG_LINKS([ \
   t/fluxometer.lua:t/fluxometer.lua \
 ])

src/bindings/python/.pylintrc

@@ -44,7 +44,7 @@ symbols=no
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable= missing-docstring, too-few-public-methods, too-many-arguments, star-args, arguments-differ, no-member, import-error, no-name-in-module
+disable=missing-docstring, too-few-public-methods, too-many-arguments, star-args, arguments-differ, no-member, import-error, no-name-in-module, useless-object-inheritance, fixme
 
 
 [REPORTS]

src/bindings/python/Makefile.am

@@ -1,12 +1,7 @@
 
-SUBDIRS=flux
+SUBDIRS=_flux flux
 
-EXTRA_DIST = pycotap test test_commands make_binding.py .pylintrc
+EXTRA_DIST = make_binding.py .pylintrc
 
 clean-local:
-	-rm -f test/*.pyc test_commands/*.pyc
 	-rm -f .coverage*
-
-dist-hook:
-	-rm -f test/*.pyc test_commands/*.pyc
-	-rm -f test_commands/*.log test_commands/*.trs

src/bindings/python/_flux/.gitignore

@@ -0,0 +1,2 @@
+*.py
+*.c

src/bindings/python/_flux/Makefile.am

@@ -0,0 +1,100 @@
+AM_CPPFLAGS = \
+	$(WARNING_CFLAGS) \
+	-I$(top_srcdir) -I$(top_srcdir)/src/include \
+	-I$(top_srcdir)/src/common/libflux \
+	$(ZMQ_CFLAGS) $(PYTHON_CPPFLAGS) \
+	$(CODE_COVERAGE_CFLAGS)
+
+AM_LDFLAGS = \
+	-avoid-version -module $(san_ld_zdef_flag) \
+	$(PYTHON_LDFLAGS) -Wl,-rpath,$(PYTHON_PREFIX)/lib \
+	$(CODE_COVERAGE_LIBS)
+
+MAKE_BINDING=$(top_srcdir)/src/bindings/python/make_binding.py
+SUFFIXES = _build.py
+
+common_libs = $(top_builddir)/src/common/libflux-core.la \
+	      $(top_builddir)/src/common/libflux-internal.la \
+	      $(ZMQ_LIBS)
+
+_build.py.c:
+	$(PYTHON) $*_build.py
+
+
+_core_build.py: $(MAKE_BINDING)
+	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libflux \
+		--package _flux \
+		--modname _core \
+		--add_sub '.*va_list.*|||' \
+		--ignore_header 'handle_impl' \
+		--ignore_header 'macros' \
+		--add_long_sub 'FLUX_SEC_TYPE_ALL.*\n.*\),|||FLUX_SEC_TYPE_ALL = 7,'\
+		flux.h
+
+BUILT_SOURCES= _core.c _jsc.c _kvs.c _kz.c
+fluxso_LTLIBRARIES = _core.la _kvs.la _jsc.la _kz.la
+fluxso_PYTHON = __init__.py
+
+nodist__core_la_SOURCES = _core.c
+_core_la_LIBADD = $(common_libs)
+_core_la_DEPENDENCIES = _core_build.py
+
+_jsc_build.py: $(top_srcdir)/src/common/libjsc/jstatctl.h $(MAKE_BINDING) _core_build.py
+	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libjsc \
+		--package _flux \
+		--modname _jsc \
+		--add_sub '.*va_list.*|||' \
+		--include_ffi _core_build \
+		jstatctl.h
+
+nodist__jsc_la_SOURCES = _jsc.c
+_jsc_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/common/libjsc
+_jsc_la_LIBADD = $(common_libs)
+_jsc_la_DEPENDENCIES = _jsc_build.py
+
+_kvs_build.py: $(top_srcdir)/src/common/libkvs/kvs.h $(MAKE_BINDING) _core_build.py
+	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libkvs \
+		--package _flux \
+		--modname _kvs \
+		--add_sub '.*va_list.*|||' \
+		--include_ffi _core_build \
+		kvs.h
+
+nodist__kvs_la_SOURCES = _kvs.c
+_kvs_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/common/libkvs
+_kvs_la_LIBADD = $(common_libs)
+_kvs_la_DEPENDENCIES = _kvs_build.py
+
+_kz_build.py: $(top_srcdir)/src/common/libkz/kz.h $(MAKE_BINDING) _core_build.py
+	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libkz \
+		--package _flux \
+		--modname _kz \
+		--add_sub '.*va_list.*|||' \
+		--include_ffi _core_build \
+		--add_sub '.*KZ_FLAGS_DELAYCOMMIT.*,|||KZ_FLAGS_DELAYCOMMIT = 0x0c00' \
+		kz.h
+
+nodist__kz_la_SOURCES = _kz.c $(top_srcdir)/src/common/libkz/kz.h
+_kz_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/common/libkz
+_kz_la_LIBADD = $(common_libs)
+_kz_la_DEPENDENCIES = _kz_build.py
+
+.PHONY: lib-copy
+
+lib-copy-vpath: ${fluxso_PYTHON} ${fluxso_LTLIBRARIES}
+	-echo Copying libraries to where they can be used by python in-tree
+	[ "$(top_srcdir)" != "$(top_builddir)" ] && cp $(top_srcdir)/src/bindings/python/_flux/__init__.py ./; \
+	for LIB in ${fluxso_LTLIBRARIES:la=so} ; do \
+		test -e .libs/$$LIB && \
+		$(LN_S) .libs/$$LIB ./ || true; \
+	done
+
+all-local: lib-copy-vpath
+
+# Support VPATH builds
+clean-local-vpath:
+	[ "$(top_srcdir)" != "$(top_builddir)" ] && rm -f $(top_builddir)/src/bindings/python/_flux/*.py || true
+
+clean-local: clean-local-vpath
+	-rm -f *.c *_build.py *.so *.pyc *.pyo
+	-rm -rf __pycache__

src/bindings/python/flux/Makefile.am

@@ -1,83 +1,3 @@
-
-AM_CPPFLAGS = \
-	$(WARNING_CFLAGS) \
-	-I$(top_srcdir) -I$(top_srcdir)/src/include \
-	-I$(top_srcdir)/src/common/libflux \
-	$(ZMQ_CFLAGS) $(PYTHON_CPPFLAGS) \
-	$(CODE_COVERAGE_CFLAGS)
-
-AM_LDFLAGS = \
-	-avoid-version -module $(san_ld_zdef_flag) \
-	$(PYTHON_LDFLAGS) -Wl,-rpath,$(PYTHON_PREFIX)/lib \
-	$(CODE_COVERAGE_LIBS)
-
-MAKE_BINDING=$(top_srcdir)/src/bindings/python/make_binding.py
-SUFFIXES = _build.py
-
-common_libs = $(top_builddir)/src/common/libflux-core.la \
-	      $(top_builddir)/src/common/libflux-internal.la \
-	      $(ZMQ_LIBS)
-
-_build.py.c:
-	$(PYTHON) $*_build.py
-
-
-_core_build.py: $(MAKE_BINDING)
-	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libflux \
-	  			  --package flux \
-	                          --modname _core \
-				  --add_sub '.*va_list.*|||' \
-				  --ignore_header 'handle_impl' \
-				  --ignore_header 'macros' \
-				  --add_long_sub 'FLUX_SEC_TYPE_ALL.*\n.*\),|||FLUX_SEC_TYPE_ALL = 7,'\
-				  flux.h
-
-fluxpy_LTLIBRARIES = _core.la _kvs.la _jsc.la _kz.la
-
-_core_la_SOURCES = _core.c
-_core_la_LIBADD = $(common_libs)
-_core_la_DEPENDENCIES = _core_build.py
-
-_jsc_build.py: $(top_srcdir)/src/common/libjsc/jstatctl.h $(MAKE_BINDING) _core_build.py
-	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libjsc \
-	  			  --package flux \
-	                          --modname _jsc \
-				  --add_sub '.*va_list.*|||' \
-				  --include_ffi _core_build \
-				  jstatctl.h
-
-_jsc_la_SOURCES = _jsc.c
-_jsc_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/common/libjsc
-_jsc_la_LIBADD = $(common_libs)
-_jsc_la_DEPENDENCIES = _jsc_build.py
-
-_kvs_build.py: $(top_srcdir)/src/common/libkvs/kvs.h $(MAKE_BINDING) _core_build.py
-	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libkvs \
-	  			  --package flux \
-	                          --modname _kvs \
-				  --add_sub '.*va_list.*|||' \
-				  --include_ffi _core_build \
-				  kvs.h
-
-_kvs_la_SOURCES = _kvs.c
-_kvs_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/common/libkvs
-_kvs_la_LIBADD = $(common_libs)
-_kvs_la_DEPENDENCIES = _kvs_build.py
-
-_kz_build.py: $(top_srcdir)/src/common/libkz/kz.h $(MAKE_BINDING) _core_build.py
-	$(PYTHON) $(MAKE_BINDING) --path $(top_srcdir)/src/common/libkz \
-				  --package flux \
-	                          --modname _kz \
-				  --add_sub '.*va_list.*|||' \
-				  --include_ffi _core_build \
-				  --add_sub '.*KZ_FLAGS_DELAYCOMMIT.*,|||KZ_FLAGS_DELAYCOMMIT = 0x0c00' \
-				  kz.h
-
-_kz_la_SOURCES = _kz.c $(top_srcdir)/src/common/libkz/kz.h
-_kz_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/common/libkz
-_kz_la_LIBADD = $(common_libs)
-_kz_la_DEPENDENCIES = _kz_build.py
-
 fluxpy_PYTHON=\
 	      __init__.py\
 	      kvs.py\
@@ -89,29 +9,15 @@ fluxpy_PYTHON=\
 	      kz.py\
 	      sec.py
 
-.PHONY: lib-copy clean-local-check
-
-lib-copy: ${fluxpy_PYTHON} ${fluxpy_LTLIBRARIES}
-	-echo Copying libraries to where they can be used by python in-tree
-	for LIB in ${fluxpy_LTLIBRARIES:la=so} ; do \
-	    cp -f .libs/$$LIB ./ ; \
-	done
-	[ "$(top_srcdir)" != "$(top_builddir)" ] && cp $(top_srcdir)/src/bindings/python/flux/*.py ./ || true
-
 if ENABLE_PYLINT
 #TODO: there must be a better way to do this
 # scan flux bindings with pylint, currently fails on any exit but 0
-check-local: lib-copy
-	if [ -x "$$( which pylint )" ] ; then  pylint --rcfile=$(top_srcdir)/src/bindings/python/.pylintrc ../flux ; fi
+check-local:
+	if [ -x "$$( which pylint )" ] ; then  pylint --rcfile=$(top_srcdir)/src/bindings/python/.pylintrc $(top_srcdir)/src/bindings/python/flux; fi
 endif
 
-all-local: lib-copy
-
-
-clean-local: clean-local-check
-	-rm -f *.c *_build.py *.so *.pyc *.pyo
-
-clean-local-check:
-	[ "$(top_srcdir)" != "$(top_builddir)" ] && rm -f $(top_builddir)/src/bindings/python/flux/*.py || true
+clean-local:
+	-rm -f *.pyc *.pyo
+	-rm -rf __pycache__
 
 SUBDIRS = core