adding deps to docker bases for python debugging

this includes black, pylint, and pyflakes, along with gdb and
strace.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.gitignore

@@ -23,6 +23,16 @@ libtool
 .libs/
 libltdl/
 
+# python environments
+env
+.eggs
+dist
+build
+flux.egg-info
+_*_clean.h
+_*_preproc.h
+*.so-built
+
 # generated library version header
 /src/common/libflux/version.h
 

configure.ac

@@ -243,22 +243,6 @@ if test -n "$PYTHON_LDFLAGS"; then
   ac_python_ldflags_set_by_user=true
 fi
 
-AM_CHECK_PYMOD(cffi,
-               [cffi.__version_info__ >= (1,1)],
-               ,
-               [AC_MSG_ERROR([could not find python module cffi, version 1.1+ required])]
-               )
-AM_CHECK_PYMOD(yaml,
-               [StrictVersion(yaml.__version__) >= StrictVersion ('3.10.0')],
-               ,
-               [AC_MSG_ERROR([could not find python module yaml, version 3.10+ required])]
-               )
-AM_CHECK_PYMOD(jsonschema,
-               [StrictVersion(jsonschema.__version__) >= StrictVersion ('2.3.0')],
-               ,
-               [AC_MSG_ERROR([could not find python module jsonschema, version 2.3.0+ required])]
-               )
-
 AS_IF([test "x$enable_docs" != "xno"], [
             AM_CHECK_PYMOD(sphinx,
                            [StrictVersion(sphinx.__version__) >= StrictVersion ('1.6.7')],
@@ -542,9 +526,6 @@ AC_CONFIG_FILES( \
   src/common/libtaskmap/Makefile \
   src/bindings/Makefile \
   src/bindings/lua/Makefile \
-  src/bindings/python/Makefile \
-  src/bindings/python/flux/Makefile \
-  src/bindings/python/_flux/Makefile \
   src/broker/Makefile \
   src/cmd/Makefile \
   src/shell/Makefile \
@@ -586,7 +567,8 @@ AC_CONFIG_FILES( \
   t/Makefile \
   t/fluxometer/conf.lua \
   t/fluxometer/conf.lua.installed \
-  src/test/docker/poison-libflux.sh
+  src/test/docker/poison-libflux.sh \
+  src/bindings/python/Makefile
 )
 
 AC_CONFIG_LINKS([ \
@@ -603,3 +585,4 @@ AS_IF([test "x$enable_docs" != "xno"], [
   fi
 ])
 
+

etc/gen-cmdhelp.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 ###############################################################
 # Copyright 2020 Lawrence Livermore National Security, LLC
 # (c.f. AUTHORS, NOTICE.LLNS, COPYING)

src/bindings/Makefile.am

@@ -1 +1,5 @@
+if WITH_PYTHON
 SUBDIRS = lua python
+else
+SUBDIRS = lua
+endif

src/bindings/python/Dockerfile

@@ -0,0 +1,69 @@
+FROM ubuntu:22.04
+
+# used in configure.ac
+ARG FLUX_VERSION=0.42.0
+ENV FLUX_VERSION=${FLUX_VERSION}
+ENV DEBIAN_FRONTEND=noninteractive
+# from root:
+# docker build -t ghcr.io/flux-framework/flux-python -f src/bindings/python/Dockerfile .
+# docker run -it ghcr.io/flux-framework/flux-python
+
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+        automake \
+        libsodium-dev \
+        libzmq3-dev \
+        libczmq-dev \
+        libjansson-dev \
+        libmunge-dev \
+        libncursesw5-dev \
+        lua5.4 \
+        liblua5.4-dev \
+        liblz4-dev \
+        libsqlite3-dev \
+        uuid-dev \
+        libhwloc-dev \
+        libmpich-dev \
+        libs3-dev \
+        libevent-dev \
+        libarchive-dev \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-sphinx \
+        libtool \
+        git \
+        build-essential \
+        # Flux security
+        libjson-glib-1.0.0 \ 
+        libjson-glib-dev \ 
+        libpam-dev && \
+        ldconfig && \
+        rm -rf /var/lib/apt/lists/*
+
+WORKDIR /code
+
+# Add flux-security directory
+RUN git clone https://github.com/flux-framework/flux-security /code/security && \
+    cd /code/security && \
+    ./autogen.sh && \    
+    ./configure --prefix=/usr/local && \
+    make && \
+    make install && \
+    ldconfig
+
+COPY . /code
+
+RUN chmod +x etc/gen-cmdhelp.py && \
+    ./autogen.sh && \
+    ./configure --prefix=/usr/local && \
+    make && \
+    make install && \
+    ldconfig
+
+# Here is how to install Python manually (and see README there)
+# WORKDIR /code/src/bindings/python
+# RUN python3 setup.py install && ldconfig
+
+WORKDIR /code
+ENTRYPOINT ["/bin/bash"]

src/bindings/python/MANIFEST.in

@@ -0,0 +1,7 @@
+include README.md
+recursive-include flux *
+prune env*
+global-exclude .env
+global-exclude *.py[co]
+recursive-exclude .git *
+global-exclude __pycache__

src/bindings/python/README.md

@@ -0,0 +1,167 @@
+# Flux Python Bindings
+
+Hello! You've found the flux Python bindings, which we currently store alongside
+the source code as they are tightly integrated. Currently, the python
+bindings build with source flux, and we are working on means to install separately.
+
+## Docker
+
+Here is a way to test installing the Python bindings. From the root of flux,
+build the [Dockerfile](Dockerfile) here. Note that we are also cloning
+flux security to show installing / building all modules.
+
+```bash
+$ docker build -t ghcr.io/flux-framework/flux-python -f src/bindings/python/Dockerfile .
+```
+
+This will build flux from your git repository, and install the Python bindings
+with a setup.py install! You can then shell into the container:
+
+```bash
+$ docker run -it ghcr.io/flux-framework/flux-python
+```
+
+And flux should be compiled, and you can use ipython to import flux:
+
+```bash
+# ipython
+> import flux
+```
+
+Note that the `make install` doesn't currently include security, as this
+is a separate repository and we need to figure out how to package it also.
+You can still install it, however (and instructions are below).
+
+### Building Modules
+
+First, here is how to build (any extension below) (without install):
+
+```bash
+$ python3 setup.py build_ext
+```
+
+And how to package into a tarball:
+
+```bash
+$ python setup.py sdist
+$ ls dist/
+flux-0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.1a1-py3.10-linux-x86_64.egg  flux-0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.1a1.tar.gz
+```
+
+Any of the install commands below are shown with "install," however you can change
+any install to a build_ext to use instead.
+
+### Development Libraries
+
+You can install with development libraries:
+
+```bash
+$ pip3 install -e .[dev]
+```
+
+### Installing Modules
+
+By default, we install core flux. However, you can use the setup.py to install
+any of the following modules (in addition to the core build):
+
+#### hostlist
+
+```bash
+$ python setup.py install --hostlist
+```
+And to test:
+```python
+import flux.hostlist
+```
+
+#### idset
+
+```bash
+$ python setup.py install --idset
+```
+
+And to test:
+```python
+import flux.idset
+```
+
+#### rlist
+
+```bash
+$ python setup.py install --rlist
+```
+I think rlist means "resource list?"  So to test:
+
+```python
+import flux.resource
+```
+
+#### security
+
+For Flux Security, the security include and source directories are required, since we couldn't
+know where you built it!
+
+```bash
+$ python setup.py install --security --security-include=/usr/local/include/flux/security --security-src=/code/security
+```
+
+Or just go NUTS
+
+```bash
+$ python3 setup.py install --hostlist --rlist --idset --security --security-include=/usr/local/include/flux/security --security-src=/code/security
+```
+
+And that's it! We still have other (modules?) to compile, and can do that next.
+
+## Development
+
+It's sometimes helpful to work interactively - e.g., instead of needing to rebuild with every change,
+you can bind the python directory on your host to the container:
+
+```bash
+$ docker run -v $PWD/src/bindings/python:/code/src/bindings/python -it ghcr.io/flux-framework/flux-python
+```
+
+This means that changes you make on your host will appear in your container, and then you can
+manually run the build:
+
+```bash
+$ cd src/bindings/python
+$ python3 setup.py install
+```
+
+And install development libraries (they are also added as an extra `flux[dev]`.
+
+```bash
+$ pip3 install -r requirements-dev.txt
+```
+
+And the setup.py takes most things as variables, so you can customize variables it would be run as follows:
+
+```bash
+$ python setup.py install --flux-root=/path/to/flux
+```
+
+## Using Pip
+
+The setup.py exports environment variables for the build script to see:
+
+ - FLUX_INSTALL_ROOT: where flux was cloned
+ - FLUX_SECURITY_INCLUDE: the security includes directory (if installing with `--security`)
+ - FLUX_SECURITY_SOURCE: the source code of security.
+
+Pip doesn't seem to see environment variables, unfortunately, so if you install with pip you need
+to export them yourself (instead of providing on the command line):
+
+```bash
+# Install development requirements
+$ FLUX_INSTALL_ROOT=/code pip3 install .[dev]
+
+# Add install options
+$ FLUX_INSTALL_ROOT=/code pip3 install --install-option="--name=value" .
+```
+
+And you can see setup.py for other install options. I'm not sure this is considered good practice
+to run additional commands in the script, but I'm not comfortable refactoring into a different
+setup routine before I know what's going on, and to do that I want to keep the logic 
+relatively close to the original.