Various fixes to get pbench-server running on RHEL 8

agent/VERSION

@@ -1 +1 @@
-0.69.8
+0.69.9

agent/bench-scripts/unittests

@@ -27,7 +27,7 @@ export _testdir=${_testroot}/pbench-agent
 # Copy bench-scripts execution environment to _testroot
 export _testopt=${_testroot}/opt/pbench-agent
 res=0
-mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/lib
+mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts
 let res=res+${?}
 cp ${_tdir}/../base ${_testopt}/
 let res=res+${?}
@@ -43,7 +43,7 @@ cp ${_tdir}/templates/* ${_testopt}/bench-scripts/templates
 let res=res+${?}
 cp -a $_tdir/../util-scripts/{getconf.py,require-rpm,pbench-*} $_testopt/util-scripts/
 let res=res+$?
-cp -a $_tdir/../../lib/* $_testopt/common/lib/
+cp -a $_tdir/../../lib/* $_testopt/lib/
 let res=res+${?}
 cp -rH ${_tdir}/test-bin/* ${_testopt}/unittest-scripts/
 let res=res+${?}
@@ -76,7 +76,7 @@ PATH=$(remove_path /opt/pbench-agent/bench-scripts ${PATH})
 PATH=$(remove_path /opt/pbench-agent/util-scripts ${PATH})
 # Allows us to intercept scp, ssh, rsync, etc.
 export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/bench-scripts:${PATH}
-export PYTHONPATH=${_testopt}/common/lib:${PYTHONPATH}
+export PYTHONPATH=${_testopt}/lib:${PYTHONPATH}
 
 # Fixed timestamp output
 export _PBENCH_UNIT_TESTS=1

agent/test-requirements.txt

@@ -0,0 +1 @@
+ansible

agent/tool-scripts/unittests

@@ -171,7 +171,7 @@ function invokit {
 
     PYTHONUNBUFFERED=True \
     _testlog=${_rundir}/test-execution.log \
-    PATH=${_optdir}/util-scripts:${_optdir}/tool-scripts:${_optdir}/common/bin:${PATH} \
+    PATH=${_optdir}/util-scripts:${_optdir}/tool-scripts:${PATH} \
     ${_optdir}/tool-scripts/${name} ${args}
 }
 

agent/util-scripts/unittests

@@ -21,13 +21,13 @@ export _testdir=${_testroot}/pbench
 # Copy util-scripts execution environment to _testroot
 _testopt=${_testroot}/opt/pbench-agent
 res=0
-mkdir -p ${_testopt}/config ${_testopt}/util-scripts ${_testopt}/tool-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/lib
+mkdir -p ${_testopt}/config ${_testopt}/util-scripts ${_testopt}/tool-scripts ${_testopt}/lib ${_testopt}/unittest-scripts
 let res=res+${?}
 cp -L ${_tdir}/../base ${_testopt}/
 let res=res+${?}
 cp -rL ${_tdir}/../config/* ${_testopt}/config/
 let res=res+${?}
-cp -a $_tdir/../../lib/* $_testopt/common/lib/
+cp -a $_tdir/../../lib/* $_testopt/lib/
 let res=res+${?}
 cp -a ${_tdir}/{getconf.py,require-rpm,pbench-*} ${_testopt}/util-scripts/
 let res=res+${?}
@@ -86,7 +86,7 @@ PATH=$(remove_path /opt/pbench-agent/bench-scripts ${PATH})
 PATH=$(remove_path /opt/pbench-agent/util-scripts ${PATH})
 # Allows us to intercept scp, ssh, rsync, etc.
 export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/tool-scripts:${PATH}
-export PYTHONPATH=${_testopt}/common/lib:${PYTHONPATH}
+export PYTHONPATH=${_testopt}/lib:${PYTHONPATH}
 
 # Fixed timestamp output
 export _PBENCH_UNIT_TESTS=1

server/Makefile

@@ -64,7 +64,7 @@ binlinks = \
 	install-build-artifacts
 
 install: install-dirs install-bin install-lib
-	${COPY} VERSION ${DESTDIR}
+	${COPY} VERSION ${DESTDIR}/
 
 install-dirs:
 	${INSTALL} ${INSTALLOPTS} ${DESTDIR}
@@ -77,37 +77,38 @@ install-bin: install-binbase install-bintools install-binscripts install-configt
 
 install-binbase:
 	for f in ${binbase} ;do \
-	    ${COPY} bin/$$f ${BINDIR} \
+	    ${COPY} bin/$$f ${BINDIR}/ \
 	;done
 
 install-bintools:
 	for f in ${bintools} ;do \
-	    ${COPY} bin/$$f ${BINDIR} \
+	    ${COPY} bin/$$f ${BINDIR}/ \
 	;done
 
 install-binscripts:
 	cd bin; \
 	for f in ${binlinks} ;do \
-	    ${COPY} $$f ${BINDIR}; \
+	    ${COPY} $$f ${BINDIR}/; \
 	    if [ -e $$f.sh ] ;then \
-	        ${COPY} $$f.sh ${BINDIR} \
+	        ${COPY} $$f.sh ${BINDIR}/ \
 	    ;elif [ -e $$f.py ] ;then \
-	        ${COPY} $$f.py ${BINDIR} \
+	        ${COPY} $$f.py ${BINDIR}/ \
 	    ;fi \
 	;done
 
 install-lib:
-	${COPY} -r lib ${DESTDIR}
+	${COPY} requirements.txt ${DESTDIR}/
+	${COPY} -r lib ${DESTDIR}/
 
 install-configtools:
-	${COPY} -r ../lib/configtools ${LIBDIR}
+	${COPY} -r ../lib/configtools ${LIBDIR}/
 
 # SHA1 and SEQNO - these are used when building an RPM only
 # so we provide a target for the spec file to invoke. This
 # is *NOT* meant to be invoked interactively.
 install-build-artifacts:
-	${COPY} ../SHA1 ${DESTDIR}
-	${COPY} ../SEQNO ${DESTDIR}
+	${COPY} ../SHA1 ${DESTDIR}/
+	${COPY} ../SEQNO ${DESTDIR}/
 
 clean:
 	rm -rf ${DESTDIR}

server/VERSION

@@ -1 +1 @@
-0.69.5
+0.69.6

server/ansible/pbench-repo-install.yml

@@ -1,11 +1,9 @@
 ---
 # ASSUMPTION: any necessary boostrapping has already been done.
-- name: install pbench.repo
+- name: install pbench RPM repo
   hosts: servers
-  remote_user: root
   become: yes
   become_user: root
 
   roles:
     - pbench-repo-install
-

server/ansible/pbench-server-install.yml

@@ -1,7 +1,9 @@
 ---
+- name: install pbench RPM repo
+  import_playbook: pbench-repo-install.yml
+
 - name: install pbench-server
   hosts: servers
-  remote_user: root
   become: yes
   become_user: root
 
@@ -11,6 +13,5 @@
     apache_options: "+Indexes +FollowSymLinks"
 
   roles:
-    - pbench-repo-install
     - pbench-server-install
     - pbench-server-config

server/bin/pbench-trampoline

@@ -3,18 +3,17 @@
 _realpath=$(realpath ${0})
 _dirname=$(dirname ${_realpath})
 _basename=$(basename ${0})
-if [[ ! ":$PATH:" =~ ":${_dirname}:" ]]; then
-   PATH=${_dirname}${PATH:+:}$PATH; export PATH
+if [[ ! ":${PATH}:" =~ ":${_dirname}:" ]]; then
+   PATH=${_dirname}${PATH:+:}${PATH}; export PATH
 fi
 if command -v python3 > /dev/null 2>&1 ;then
     _pythoncmd=""
-elif command -v scl > /dev/null 2>&1 ;then
-    _pythoncmd="scl enable rh-python36 --"
 else
-    printf -- "${_basename}: pbench-trampoline needs python3, either directly or through SCL\n" >&2
+    printf -- "${_basename}: pbench-trampoline needs python3\n" >&2
     exit 1
 fi
-export PYTHONPATH=$(dirname ${_dirname})/lib:$(dirname ${_dirname})/common/lib:$PYTHONPATH
+_installdir=$(dirname ${_dirname})
+export PYTHONPATH=${_installdir}/lib:${_installdir}/lib/python3.8/site-packages:${PYTHONPATH}
 if [[ -e ${_dirname}/${_basename}.py ]]; then
     # We don't use pbench-base.py to invoke python programs, they can use the
     # PbenchConfig class directly.

server/bin/unittests

@@ -254,8 +254,6 @@ function _setup_state {
     let res=res+$?
     mkdir -p $_testopt_sat/bin
     let res=res+$?
-    mkdir -p $_testopt/common/lib
-    let res=res+$?
     cp -a $_tdir/{unittests,getconf.py,pbench*} $_testopt/bin
     let res=res+$?
     mkdir -p $_testopt/html/static/{js,css}/v0.{2,3}
@@ -270,12 +268,14 @@ function _setup_state {
     let res=res+$?
     cp -a $_tdir/{unittests,getconf.py,pbench*} $_testopt_sat/bin
     let res=res+$?
-    cp -a $_tdir/../../lib/* $_testopt/common/lib
-    let res=res+$?
     cp -a $_tdir/../lib $_testopt
     let res=res+$?
+    cp -a $_tdir/../../lib/* $_testopt/lib
+    let res=res+$?
     cp -a $_tdir/../lib $_testopt_sat
     let res=res+$?
+    cp -a $_tdir/../../lib/* $_testopt_sat/lib
+    let res=res+$?
     mkdir -p $_testopt_sat/html/static/{js,css}/v0.{2,3}
     let res=res+$?
     cp -a $_tdir/../../web-server/v0.2/js/ $_testopt_sat/html/static/js/v0.2
@@ -317,7 +317,7 @@ function _setup_state {
     # are found in $_testopt/unittest-scripts.
     _orig_PATH=$PATH
     export PATH=$_testopt/unittest-scripts:$_testopt/bin:$PATH
-    export PYTHONPATH="${_testopt}/lib:${_testopt}/common/lib:${PYTHONPATH}"
+    export PYTHONPATH="${_testopt}/lib:${PYTHONPATH}"
 
     # Expected location of the final configuration files
     export _PBENCH_SERVER_CONFIG=$_testopt/lib/config/pbench-server.cfg

server/requirements.txt

@@ -0,0 +1,2 @@
+boto3
+elasticsearch1

server/rpm/Makefile

@@ -59,12 +59,10 @@ tarball: spec
 
 ###########################################################################
 # build RPMs in COPR
-seqno := $(shell if [ -f seqno ] ;then cat seqno ;else echo 1; fi)
 
 include rpm.mk
 
 clean-sha1:
 	rm -f ${upstmtree}/${prog}.SHA1
 
 clean:: localclean
-

server/rpm/pbench-server.spec.j2

@@ -8,30 +8,14 @@ URL:            http://github.com/distributed-systems-analysis/pbench
 Source0:        pbench-server-%{version}.tar.gz
 Buildarch:      noarch
 
-
-%if 0%{?rhel} == 7
-# need python3 - install scl to get it
-# scl-utils is in the main repo, but the scl python3
-# collection du jour is in the scl repo which
-# is assumed to have been added already with:
-#    yum install yum-utils
-#    yum-config-manager --enable rhel-server-rhscl-7-rpms
-#
-# we can't do that in the spec file, because of the yum lock,
-# so they have to be done beforehand.
-Requires: scl-utils, rh-python36
-%else
-Requires: python3 python3-devel gcc
-%endif
+# NOTE: we add the "six" and "urllib3" Python 3.8 RPM requirements so that
+# we don't end up pip installing those modules in the /opt/pbench-server/lib
+# tree.
+Requires: python38 python38-six python38-urllib3
 
 # policycoreutils for semanage and restorecon - used in pbench-server-activate-create-results-dir
-Requires:       policycoreutils
-%if 0%{?rhel} != 7
+Requires: policycoreutils
 Requires: policycoreutils-python-utils
-%else
-Requires: policycoreutils-python
-%endif
-
 Requires: npm
 
 # installdir has to agree with the definition of install-dir in
@@ -70,14 +54,7 @@ mv %{buildroot}/%{installdir}/%{static}/package.json %{buildroot}/%{installdir}
 
 %post
 # Install python dependencies
-%if 0%{?rhel} == 7
-# install python3-elasticsearch
-echo "scl enable rh-python36 'pip install elasticsearch1 boto3 ujson'"
-scl enable rh-python36 'pip install elasticsearch1 boto3 ujson'
-%else
-echo "pip3 install -r /%{installdir}/requirements.txt"
-pip3 install -r /%{installdir}/requirements.txt
-%endif
+pip3 --no-cache-dir install --prefix=/%{installdir} -r /%{installdir}/requirements.txt > /%{installdir}/pip3-install.log
 
 # install node.js modules under /%{installdir}
 cd /%{installdir}
@@ -122,6 +99,7 @@ fi
 /%{installdir}/VERSION
 /%{installdir}/SEQNO
 /%{installdir}/SHA1
+/%{installdir}/requirements.txt
 /%{installdir}/%{static}/VERSION
 /%{installdir}/package.json
 

tox.ini

@@ -10,9 +10,6 @@ usedevelop = True
 install_command = pip install -U {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
-deps =
-   -r{toxinidir}/requirements.txt
-   -r{toxinidir}/test-requirements.txt
 envars =
    PATH = {env:PATH}:{toxinidir}/bin
 whitelist_externals =
@@ -22,15 +19,19 @@ whitelist_externals =
     env
 
 [testenv:server]
-usedevelop = true
 passenv = PBENCH_UNITTEST_SERVER_MODE
+deps =
+   -r{toxinidir}/server/requirements.txt
+   -r{toxinidir}/test-requirements.txt
 commands_pre = 
   find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -path '*/__pycache__/*' -name '*.py[c|o]' -delete
 commands =
   bash -c "./server/bin/unittests {posargs}"
 
 [testenv:agent]
-usedevelop = true
+deps =
+   -r{toxinidir}/test-requirements.txt
+   -r{toxinidir}/agent/test-requirements.txt
 commands_pre = 
   find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -path '*/__pycache__/*' -name '*.py[c|o]' -delete
 commands =