Clean build (#3)

* fix bug with missing sdist_info, just do same for all commands
* prepare to test manual deploy

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/scripts/setup.sh

@@ -3,8 +3,12 @@
 set -euo pipefail
 
 # This will be empty for nightly test, and we will clone master branch
-FLUX_VERSION=${FLUX_VERSION:-0.48.0}
-echo "Flux Version is ${FLUX_VERSION}"
+FLUX_RELEASE_VERSION=${FLUX_RELEASE_VERSION:-0.50.0}
+FLUX_VERSION=${FLUX_VERSION:-develop}
+
+# Prepare the version file
+echo "Flux Version for pypi is ${FLUX_VERSION}"
+sed -i "s/package_version = \"develop\"/package_version = \"$FLUX_VERSION\"/" setup.py
 
 here=$(pwd)
 sudo apt-get update
@@ -53,25 +57,25 @@ sudo make install
 sudo ldconfig
 
 # This is a build from a release 
-if [[ "${FLUX_VERSION}" != "" ]]; then
-    echo "Flux version requested to build is ${FLUX_VERSION}"
-    wget https://github.com/flux-framework/flux-core/releases/download/v${FLUX_VERSION}/flux-core-${FLUX_VERSION}.tar.gz
-    tar -xzvf flux-core-${FLUX_VERSION}.tar.gz
-    sudo mv flux-core-${FLUX_VERSION} /code
+echo "Flux Release Version is ${FLUX_RELEASE_VERSION}"
+echo "Flux version requested to build is ${FLUX_RELEASE_VERSION}"
+wget https://github.com/flux-framework/flux-core/releases/download/v${FLUX_RELEASE_VERSION}/flux-core-${FLUX_RELEASE_VERSION}.tar.gz
+tar -xzvf flux-core-${FLUX_RELEASE_VERSION}.tar.gz
+sudo mv flux-core-${FLUX_RELEASE_VERSION} /code
 
-# Build from the current master branch
-else
-    sudo git clone https://github.com/flux-framework/flux-core /code
-fi
 sudo chown -R $USER /code
 cd /code
 chmod +x etc/gen-cmdhelp.py
 
 # This is only needed for non-releases
 ./autogen.sh || echo "No autogen here"
 ./configure --prefix=/usr/local
-make
-sudo make install
+
+# We don't really care about the version here -just building python bindings
+make VERBOSE=1
+
+# This sometimes fails the first time but then works ok? Weird
+sudo make install || true
 sudo make install
 sudo ldconfig   
 cd ${here}

.github/workflows/build-legacy.yaml

@@ -0,0 +1,66 @@
+name: build-legacy flux-python
+
+on:
+  pull_request: []
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Pypi version'
+      branch:
+        description: 'Branch to build from'
+        default: "0.49.0"
+      repo:
+        description: 'Repository to build from'
+        default: "https://github.com/rse-ops/flux-core-python"
+
+jobs:
+  build-manual:
+    runs-on: ubuntu-latest
+    services:
+      api:
+        image: fluxrm/testenv:focal
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Build Flux Core Branch
+      env:
+        FLUX_RELEASE_VERSION: ${{ inputs.release_version }}
+        FLUX_VERSION: ${{ inputs.version }}
+      run: /bin/bash .github/scripts/setup.sh
+
+    - name: Build Python Bindings
+      env:
+        FLUX_BRANCH: ${{ inputs.branch }}
+        FLUX_REPO: ${{ inputs.repo }}
+      run: |
+        if [[ "${FLUX_REPO}" == "" ]]; then
+            FLUX_REPO="https://github.com/flux-framework/flux-core"
+        fi
+        if [[ "${FLUX_BRANCH}" == "" ]]; then
+            FLUX_BRANCH="master"
+        fi
+        git clone -b ${FLUX_BRANCH} ${FLUX_REPO} /tmp/flux-core   
+        mv /tmp/flux-core/src/bindings/python/flux ./flux
+        python3 setup.py sdist
+        sudo ldconfig
+    
+    - name: Prepare Wheel
+      if: (github.event_name != 'pull_request')
+      run: conda create --quiet --name env twine
+
+    - name: Install dependencies
+      if: (github.event_name != 'pull_request')
+      run: |
+        export PATH="/usr/share/miniconda/bin:$PATH"
+        source activate env
+        pip install setuptools wheel twine
+
+    - name: Build and publish
+      if: (github.event_name != 'pull_request')
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USER }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASS }}
+      run: |
+        export PATH="/usr/share/miniconda/bin:$PATH"
+        source activate env
+        twine upload dist/*.tar.gz

.github/workflows/build-manual.yaml

@@ -4,8 +4,16 @@ on:
   workflow_dispatch:
     inputs:
       version:
+        description: 'Pypi version'
+      release_version:
         description: 'Version of flux to build'
-        required: true
+        default: "0.50.0"
+      branch:
+        description: 'Branch to build from'
+        default: "master"
+      repo:
+        description: 'Repository to build from'
+        default: "https://github.com/flux-framework/flux-core"
 
 jobs:
   build-manual:
@@ -16,36 +24,46 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Build Flux Core
+    - name: Build Flux Core Branch
       env:
+        FLUX_RELEASE_VERSION: ${{ inputs.release_version }}
         FLUX_VERSION: ${{ inputs.version }}
-      run: |
-        export FLUX_VERSION
-        /bin/bash .github/scripts/setup.sh
+      run: /bin/bash .github/scripts/setup.sh
 
     - name: Build Python Bindings
+      env:
+        FLUX_BRANCH: ${{ inputs.branch }}
+        FLUX_REPO: ${{ inputs.repo }}
       run: |
-        ls /code   
-        ls ./            
-        mv /code/src/bindings/python/flux ./flux
+        if [[ "${FLUX_REPO}" == "" ]]; then
+            FLUX_REPO="https://github.com/flux-framework/flux-core"
+        fi
+        if [[ "${FLUX_BRANCH}" == "" ]]; then
+            FLUX_BRANCH="master"
+        fi
+        git clone -b ${FLUX_BRANCH} ${FLUX_REPO} /tmp/flux-core   
+        mv /tmp/flux-core/src/bindings/python/flux ./flux
         python3 setup.py sdist
         sudo ldconfig
     
     - name: Prepare Wheel
+      if: (github.event_name != 'pull_request')
       run: conda create --quiet --name env twine
 
     - name: Install dependencies
+      if: (github.event_name != 'pull_request')
       run: |
         export PATH="/usr/share/miniconda/bin:$PATH"
         source activate env
-        pip install -e .
         pip install setuptools wheel twine
 
     - name: Build and publish
+      if: (github.event_name != 'pull_request')
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USER }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASS }}
       run: |
         export PATH="/usr/share/miniconda/bin:$PATH"
         source activate env
         twine upload dist/*.tar.gz
+

.github/workflows/test-build.yaml

@@ -15,9 +15,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Build Flux Core
+      env:
+        FLUX_RELEASE_VERSION: "0.50.0"
       run: /bin/bash .github/scripts/setup.sh
 
     - name: Build Python Bindings
       run: |
         mv /code/src/bindings/python/flux ./flux
-        python3 setup.py sdist
+        python3 setup.py sdist

setup.py

@@ -25,10 +25,10 @@
 
 # Metadata
 package_name = "flux-python"
-package_version = "0.48.0-rc2"
+package_version = "develop"
 package_description = "Python bindings for the flux resource manager API"
 package_url = "https://github.com/flux-framework/flux-python"
-package_keywords = "flux, job manager, orchestration, hpc"
+package_keywords = "flux, job manager, workload manager, orchestration, hpc"
 
 try:
     with open("README.md") as filey:
@@ -43,29 +43,32 @@
 root = os.path.dirname(os.path.abspath(__file__))
 source = os.path.join(root, "src")
 
+
 def find_flux():
     """
     Find flux install via the executable!
     """
-    path = shutil.which('flux')
+    path = shutil.which("flux")
     if not path:
-        sys.exit('Cannot find executable flux, which is required to be on PATH to find the install location.')
+        sys.exit(
+            "Cannot find executable flux, which is required to be on PATH to find the install location."
+        )
     # /usr/local/bin/flux --> /usr/local
     return os.path.dirname(os.path.dirname(path))
 
+
 flux_root = find_flux()
 
 # Module specific default options files. Format strings below will be populated
 # after receiving the custom varibles from the user
 options = {
     "core": {
         "header": "include/flux/core.h",
-        "additional_headers": [os.path.join(source, "callbacks.h")]
+        "additional_headers": [os.path.join(source, "callbacks.h")],
     },
     "hostlist": {
         "header": "include/flux/hostlist.h",
     },
-
     # Note that rlist is currently disabled, so this
     # set of metadata doesn't matter
     "rlist": {
@@ -88,6 +91,7 @@ def find_flux():
 # rlist.h is disabled for now, as it requires the flux-core build
 # build_types = {"core", "idset", "rlist", "security", "hostlist"}
 
+
 @contextmanager
 def workdir(dirname):
     """
@@ -161,7 +165,6 @@ def _parse_comma_list(self, attr):
             value = []
         setattr(self, attr, value)
 
-
     def run(self):
         """
         Run the install
@@ -196,7 +199,11 @@ def __init__(self, root, build_type, **kwargs):
         self.header = kwargs["header"]
 
         # Update search to include defaults
-        custom_search = [os.path.join(self.root, "include"), os.path.join(self.root, "lib"), os.path.join(self.root, "include", "flux")]
+        custom_search = [
+            os.path.join(self.root, "include"),
+            os.path.join(self.root, "lib"),
+            os.path.join(self.root, "include", "flux"),
+        ]
         self.search = custom_search + [
             x.format(root=root) for x in kwargs.get("search", [])
         ]
@@ -254,14 +261,14 @@ def preprocess_gcc(self):
         if not gcc:
             sys.exit("Cannot find gcc compiler.")
         cmd = [
-              gcc,
-              "-E",
-              "-D __attribute__(...)=",
-              '-DFLUX_DEPRECATED(...)=',
-              self.output,
-              "-o",
-              self.preproc_output,
-          ]
+            gcc,
+            "-E",
+            "-D __attribute__(...)=",
+            "-DFLUX_DEPRECATED(...)=",
+            self.output,
+            "-o",
+            self.preproc_output,
+        ]
         print(" ".join(cmd))
         res = subprocess.call(cmd)
         if res != 0:
@@ -317,6 +324,7 @@ def check_header(self, f, including_path="."):
         # Flag as checked
         self.checked_heads[f] = 1
 
+
 def setup():
     """
     A wrapper to run setup. This likely isn't best practice, but is a first effort.
@@ -330,14 +338,13 @@ def setup():
     # The flux security path should be in the same root, under includes
     security_include = os.path.join(flux_root, "include", "flux", "security")
     if not os.path.exists(security_include):
-        sys.exit(f'Cannot find flux security under expected path {security_include}')
+        sys.exit(f"Cannot find flux security under expected path {security_include}")
 
-    # We only want this to run on creating the tarball
+    # We only want this to run on creating the tarball or install
     command = sys.argv[1]
-    if command in ["sdist", "build", "build_ext"]:
-        # Custom setup commands, first without cffi to prepare headers
-        prepare = PrepareFluxHeaders(flux_root)
-        prepare.run()
+    print(f"Command is {command}")
+    prepare = PrepareFluxHeaders(flux_root)
+    prepare.run()
 
     # Request to install additional modules (we always do core0
     # We also have to remove the setup.py flags that aren't known
@@ -364,7 +371,7 @@ def setup():
         packages=find_packages(),
         include_package_data=True,
         zip_safe=False,
-        install_requires=[cffi_dep],
+        install_requires=[cffi_dep, "pyyaml"],
         extras_require={
             "dev": ["pyyaml", "jsonschema", "docutils", "black", "IPython"]
         },