From 948f58f372218dfe300c80510abb43e5efad75b3 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 8 Jun 2021 01:25:51 -0400 Subject: [PATCH] Various fixes to get pbench-server running on RHEL 8 --- agent/VERSION | 2 +- agent/bench-scripts/unittests | 6 ++-- agent/test-requirements.txt | 1 + agent/tool-scripts/unittests | 2 +- agent/util-scripts/unittests | 6 ++-- requirements.txt | 4 --- server/Makefile | 21 +++++++------- server/VERSION | 2 +- server/ansible/pbench-repo-install.yml | 4 +-- server/ansible/pbench-server-install.yml | 5 ++-- server/bin/pbench-trampoline | 11 ++++---- server/bin/unittests | 10 +++---- server/requirements.txt | 2 ++ server/rpm/Makefile | 2 -- server/rpm/pbench-server.spec.j2 | 36 +++++------------------- tox.ini | 11 ++++---- 16 files changed, 50 insertions(+), 75 deletions(-) create mode 100644 agent/test-requirements.txt delete mode 100644 requirements.txt create mode 100644 server/requirements.txt diff --git a/agent/VERSION b/agent/VERSION index d1a1643813..98211a99a9 100644 --- a/agent/VERSION +++ b/agent/VERSION @@ -1 +1 @@ -0.69.8 +0.69.9 diff --git a/agent/bench-scripts/unittests b/agent/bench-scripts/unittests index 44b2f0d193..a602ebd2cb 100755 --- a/agent/bench-scripts/unittests +++ b/agent/bench-scripts/unittests @@ -27,7 +27,7 @@ export _testdir=${_testroot}/pbench-agent # Copy bench-scripts execution environment to _testroot export _testopt=${_testroot}/opt/pbench-agent res=0 -mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/lib +mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts let res=res+${?} cp ${_tdir}/../base ${_testopt}/ let res=res+${?} @@ -43,7 +43,7 @@ cp ${_tdir}/templates/* ${_testopt}/bench-scripts/templates let res=res+${?} cp -a $_tdir/../util-scripts/{getconf.py,require-rpm,pbench-*} $_testopt/util-scripts/ let res=res+$? -cp -a $_tdir/../../lib/* $_testopt/common/lib/ +cp -a $_tdir/../../lib/* $_testopt/lib/ let res=res+${?} cp -rH ${_tdir}/test-bin/* ${_testopt}/unittest-scripts/ let res=res+${?} @@ -76,7 +76,7 @@ PATH=$(remove_path /opt/pbench-agent/bench-scripts ${PATH}) PATH=$(remove_path /opt/pbench-agent/util-scripts ${PATH}) # Allows us to intercept scp, ssh, rsync, etc. export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/bench-scripts:${PATH} -export PYTHONPATH=${_testopt}/common/lib:${PYTHONPATH} +export PYTHONPATH=${_testopt}/lib:${PYTHONPATH} # Fixed timestamp output export _PBENCH_UNIT_TESTS=1 diff --git a/agent/test-requirements.txt b/agent/test-requirements.txt new file mode 100644 index 0000000000..90d40550bc --- /dev/null +++ b/agent/test-requirements.txt @@ -0,0 +1 @@ +ansible diff --git a/agent/tool-scripts/unittests b/agent/tool-scripts/unittests index 1c68770aaf..c1ae0947d5 100755 --- a/agent/tool-scripts/unittests +++ b/agent/tool-scripts/unittests @@ -171,7 +171,7 @@ function invokit { PYTHONUNBUFFERED=True \ _testlog=${_rundir}/test-execution.log \ - PATH=${_optdir}/util-scripts:${_optdir}/tool-scripts:${_optdir}/common/bin:${PATH} \ + PATH=${_optdir}/util-scripts:${_optdir}/tool-scripts:${PATH} \ ${_optdir}/tool-scripts/${name} ${args} } diff --git a/agent/util-scripts/unittests b/agent/util-scripts/unittests index 4d57ad4556..26a88b1357 100755 --- a/agent/util-scripts/unittests +++ b/agent/util-scripts/unittests @@ -21,13 +21,13 @@ export _testdir=${_testroot}/pbench # Copy util-scripts execution environment to _testroot _testopt=${_testroot}/opt/pbench-agent res=0 -mkdir -p ${_testopt}/config ${_testopt}/util-scripts ${_testopt}/tool-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/lib +mkdir -p ${_testopt}/config ${_testopt}/util-scripts ${_testopt}/tool-scripts ${_testopt}/lib ${_testopt}/unittest-scripts let res=res+${?} cp -L ${_tdir}/../base ${_testopt}/ let res=res+${?} cp -rL ${_tdir}/../config/* ${_testopt}/config/ let res=res+${?} -cp -a $_tdir/../../lib/* $_testopt/common/lib/ +cp -a $_tdir/../../lib/* $_testopt/lib/ let res=res+${?} cp -a ${_tdir}/{getconf.py,require-rpm,pbench-*} ${_testopt}/util-scripts/ let res=res+${?} @@ -86,7 +86,7 @@ PATH=$(remove_path /opt/pbench-agent/bench-scripts ${PATH}) PATH=$(remove_path /opt/pbench-agent/util-scripts ${PATH}) # Allows us to intercept scp, ssh, rsync, etc. export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/tool-scripts:${PATH} -export PYTHONPATH=${_testopt}/common/lib:${PYTHONPATH} +export PYTHONPATH=${_testopt}/lib:${PYTHONPATH} # Fixed timestamp output export _PBENCH_UNIT_TESTS=1 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e95f82b03e..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -ansible -boto3 -elasticsearch -sh diff --git a/server/Makefile b/server/Makefile index 2777eb7542..c6ade70fcd 100644 --- a/server/Makefile +++ b/server/Makefile @@ -64,7 +64,7 @@ binlinks = \ install-build-artifacts install: install-dirs install-bin install-lib - ${COPY} VERSION ${DESTDIR} + ${COPY} VERSION ${DESTDIR}/ install-dirs: ${INSTALL} ${INSTALLOPTS} ${DESTDIR} @@ -77,37 +77,38 @@ install-bin: install-binbase install-bintools install-binscripts install-configt install-binbase: for f in ${binbase} ;do \ - ${COPY} bin/$$f ${BINDIR} \ + ${COPY} bin/$$f ${BINDIR}/ \ ;done install-bintools: for f in ${bintools} ;do \ - ${COPY} bin/$$f ${BINDIR} \ + ${COPY} bin/$$f ${BINDIR}/ \ ;done install-binscripts: cd bin; \ for f in ${binlinks} ;do \ - ${COPY} $$f ${BINDIR}; \ + ${COPY} $$f ${BINDIR}/; \ if [ -e $$f.sh ] ;then \ - ${COPY} $$f.sh ${BINDIR} \ + ${COPY} $$f.sh ${BINDIR}/ \ ;elif [ -e $$f.py ] ;then \ - ${COPY} $$f.py ${BINDIR} \ + ${COPY} $$f.py ${BINDIR}/ \ ;fi \ ;done install-lib: - ${COPY} -r lib ${DESTDIR} + ${COPY} requirements.txt ${DESTDIR}/ + ${COPY} -r lib ${DESTDIR}/ install-configtools: - ${COPY} -r ../lib/configtools ${LIBDIR} + ${COPY} -r ../lib/configtools ${LIBDIR}/ # SHA1 and SEQNO - these are used when building an RPM only # so we provide a target for the spec file to invoke. This # is *NOT* meant to be invoked interactively. install-build-artifacts: - ${COPY} ../SHA1 ${DESTDIR} - ${COPY} ../SEQNO ${DESTDIR} + ${COPY} ../SHA1 ${DESTDIR}/ + ${COPY} ../SEQNO ${DESTDIR}/ clean: rm -rf ${DESTDIR} diff --git a/server/VERSION b/server/VERSION index c482845a51..0f576b6872 100644 --- a/server/VERSION +++ b/server/VERSION @@ -1 +1 @@ -0.69.5 +0.69.6 diff --git a/server/ansible/pbench-repo-install.yml b/server/ansible/pbench-repo-install.yml index 5aa77544c6..f3506f6147 100644 --- a/server/ansible/pbench-repo-install.yml +++ b/server/ansible/pbench-repo-install.yml @@ -1,11 +1,9 @@ --- # ASSUMPTION: any necessary boostrapping has already been done. -- name: install pbench.repo +- name: install pbench RPM repo hosts: servers - remote_user: root become: yes become_user: root roles: - pbench-repo-install - diff --git a/server/ansible/pbench-server-install.yml b/server/ansible/pbench-server-install.yml index 76ac63f712..f5b9c248f7 100644 --- a/server/ansible/pbench-server-install.yml +++ b/server/ansible/pbench-server-install.yml @@ -1,7 +1,9 @@ --- +- name: install pbench RPM repo + import_playbook: pbench-repo-install.yml + - name: install pbench-server hosts: servers - remote_user: root become: yes become_user: root @@ -11,6 +13,5 @@ apache_options: "+Indexes +FollowSymLinks" roles: - - pbench-repo-install - pbench-server-install - pbench-server-config diff --git a/server/bin/pbench-trampoline b/server/bin/pbench-trampoline index c4cc471a7c..f006904de1 100755 --- a/server/bin/pbench-trampoline +++ b/server/bin/pbench-trampoline @@ -3,18 +3,17 @@ _realpath=$(realpath ${0}) _dirname=$(dirname ${_realpath}) _basename=$(basename ${0}) -if [[ ! ":$PATH:" =~ ":${_dirname}:" ]]; then - PATH=${_dirname}${PATH:+:}$PATH; export PATH +if [[ ! ":${PATH}:" =~ ":${_dirname}:" ]]; then + PATH=${_dirname}${PATH:+:}${PATH}; export PATH fi if command -v python3 > /dev/null 2>&1 ;then _pythoncmd="" -elif command -v scl > /dev/null 2>&1 ;then - _pythoncmd="scl enable rh-python36 --" else - printf -- "${_basename}: pbench-trampoline needs python3, either directly or through SCL\n" >&2 + printf -- "${_basename}: pbench-trampoline needs python3\n" >&2 exit 1 fi -export PYTHONPATH=$(dirname ${_dirname})/lib:$(dirname ${_dirname})/common/lib:$PYTHONPATH +_installdir=$(dirname ${_dirname}) +export PYTHONPATH=${_installdir}/lib:${_installdir}/lib/python3.8/site-packages:${PYTHONPATH} if [[ -e ${_dirname}/${_basename}.py ]]; then # We don't use pbench-base.py to invoke python programs, they can use the # PbenchConfig class directly. diff --git a/server/bin/unittests b/server/bin/unittests index 70a8156fd9..0f2016ab28 100755 --- a/server/bin/unittests +++ b/server/bin/unittests @@ -254,8 +254,6 @@ function _setup_state { let res=res+$? mkdir -p $_testopt_sat/bin let res=res+$? - mkdir -p $_testopt/common/lib - let res=res+$? cp -a $_tdir/{unittests,getconf.py,pbench*} $_testopt/bin let res=res+$? mkdir -p $_testopt/html/static/{js,css}/v0.{2,3} @@ -270,12 +268,14 @@ function _setup_state { let res=res+$? cp -a $_tdir/{unittests,getconf.py,pbench*} $_testopt_sat/bin let res=res+$? - cp -a $_tdir/../../lib/* $_testopt/common/lib - let res=res+$? cp -a $_tdir/../lib $_testopt let res=res+$? + cp -a $_tdir/../../lib/* $_testopt/lib + let res=res+$? cp -a $_tdir/../lib $_testopt_sat let res=res+$? + cp -a $_tdir/../../lib/* $_testopt_sat/lib + let res=res+$? mkdir -p $_testopt_sat/html/static/{js,css}/v0.{2,3} let res=res+$? cp -a $_tdir/../../web-server/v0.2/js/ $_testopt_sat/html/static/js/v0.2 @@ -317,7 +317,7 @@ function _setup_state { # are found in $_testopt/unittest-scripts. _orig_PATH=$PATH export PATH=$_testopt/unittest-scripts:$_testopt/bin:$PATH - export PYTHONPATH="${_testopt}/lib:${_testopt}/common/lib:${PYTHONPATH}" + export PYTHONPATH="${_testopt}/lib:${PYTHONPATH}" # Expected location of the final configuration files export _PBENCH_SERVER_CONFIG=$_testopt/lib/config/pbench-server.cfg diff --git a/server/requirements.txt b/server/requirements.txt new file mode 100644 index 0000000000..b13dc1f26b --- /dev/null +++ b/server/requirements.txt @@ -0,0 +1,2 @@ +boto3 +elasticsearch1 diff --git a/server/rpm/Makefile b/server/rpm/Makefile index 7acc510fca..63a13fc939 100644 --- a/server/rpm/Makefile +++ b/server/rpm/Makefile @@ -59,7 +59,6 @@ tarball: spec ########################################################################### # build RPMs in COPR -seqno := $(shell if [ -f seqno ] ;then cat seqno ;else echo 1; fi) include rpm.mk @@ -67,4 +66,3 @@ clean-sha1: rm -f ${upstmtree}/${prog}.SHA1 clean:: localclean - diff --git a/server/rpm/pbench-server.spec.j2 b/server/rpm/pbench-server.spec.j2 index 16ffc52658..d388df33cf 100644 --- a/server/rpm/pbench-server.spec.j2 +++ b/server/rpm/pbench-server.spec.j2 @@ -8,30 +8,14 @@ URL: http://github.com/distributed-systems-analysis/pbench Source0: pbench-server-%{version}.tar.gz Buildarch: noarch - -%if 0%{?rhel} == 7 -# need python3 - install scl to get it -# scl-utils is in the main repo, but the scl python3 -# collection du jour is in the scl repo which -# is assumed to have been added already with: -# yum install yum-utils -# yum-config-manager --enable rhel-server-rhscl-7-rpms -# -# we can't do that in the spec file, because of the yum lock, -# so they have to be done beforehand. -Requires: scl-utils, rh-python36 -%else -Requires: python3 python3-devel gcc -%endif +# NOTE: we add the "six" and "urllib3" Python 3.8 RPM requirements so that +# we don't end up pip installing those modules in the /opt/pbench-server/lib +# tree. +Requires: python38 python38-six python38-urllib3 # policycoreutils for semanage and restorecon - used in pbench-server-activate-create-results-dir -Requires: policycoreutils -%if 0%{?rhel} != 7 +Requires: policycoreutils Requires: policycoreutils-python-utils -%else -Requires: policycoreutils-python -%endif - Requires: npm # installdir has to agree with the definition of install-dir in @@ -70,14 +54,7 @@ mv %{buildroot}/%{installdir}/%{static}/package.json %{buildroot}/%{installdir} %post # Install python dependencies -%if 0%{?rhel} == 7 -# install python3-elasticsearch -echo "scl enable rh-python36 'pip install elasticsearch1 boto3 ujson'" -scl enable rh-python36 'pip install elasticsearch1 boto3 ujson' -%else -echo "pip3 install -r /%{installdir}/requirements.txt" -pip3 install -r /%{installdir}/requirements.txt -%endif +pip3 --no-cache-dir install --prefix=/%{installdir} -r /%{installdir}/requirements.txt > /%{installdir}/pip3-install.log # install node.js modules under /%{installdir} cd /%{installdir} @@ -122,6 +99,7 @@ fi /%{installdir}/VERSION /%{installdir}/SEQNO /%{installdir}/SHA1 +/%{installdir}/requirements.txt /%{installdir}/%{static}/VERSION /%{installdir}/package.json diff --git a/tox.ini b/tox.ini index bfa787ab9f..19c80f1be9 100644 --- a/tox.ini +++ b/tox.ini @@ -10,9 +10,6 @@ usedevelop = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt envars = PATH = {env:PATH}:{toxinidir}/bin whitelist_externals = @@ -22,15 +19,19 @@ whitelist_externals = env [testenv:server] -usedevelop = true passenv = PBENCH_UNITTEST_SERVER_MODE +deps = + -r{toxinidir}/server/requirements.txt + -r{toxinidir}/test-requirements.txt commands_pre = find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -path '*/__pycache__/*' -name '*.py[c|o]' -delete commands = bash -c "./server/bin/unittests {posargs}" [testenv:agent] -usedevelop = true +deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/agent/test-requirements.txt commands_pre = find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -path '*/__pycache__/*' -name '*.py[c|o]' -delete commands =