From 1be2def46a1d5dd735d53cc8b73d8a562ef0f837 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 12 Oct 2020 13:48:22 +0200 Subject: [PATCH 1/7] Add cli option to install optional python packages --- bootstrap.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index fb0dd0ce8e..6d89770bf7 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -24,13 +24,15 @@ usage() echo "Usage: $0 [-h] [+docs]" echo "Bootstrap ReFrame by pulling all its dependencies" echo " -P EXEC Use EXEC as Python interpreter" + echo " -O Install optional packages using pip" echo " -h Print this help message and exit" echo " +docs Build also the documentation" } -while getopts "hP:" opt; do +while getopts "hOP:" opt; do case $opt in + "O") optional="pygelf" ;; "P") python=$OPTARG ;; "h") usage && exit 0 ;; "?") usage && exit 0 ;; @@ -62,6 +64,10 @@ export PYTHONPATH=$(pwd)/external:$(pwd)/external/usr/lib/python$pyver/site-pack CMD $python -m pip install --no-cache-dir -q --upgrade pip --target=external/ CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade +if [ -n "$optional" ]; then + CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q $optional --target=external/ --upgrade +fi + if [ x"$1" == x"+docs" ]; then CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r docs/requirements.txt --target=external/ --upgrade make -C docs From 6ea218d0d1554b3f29fd9ec38ad879e1b370b3e0 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 12 Oct 2020 15:49:31 +0200 Subject: [PATCH 2/7] Fix help message of optional package bootstrapping --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 6d89770bf7..8c8591b70e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -24,7 +24,7 @@ usage() echo "Usage: $0 [-h] [+docs]" echo "Bootstrap ReFrame by pulling all its dependencies" echo " -P EXEC Use EXEC as Python interpreter" - echo " -O Install optional packages using pip" + echo " -O Install optional Python packages using pip" echo " -h Print this help message and exit" echo " +docs Build also the documentation" } From b74f8ff85b2a410f6596e291b137976f8c7a9d69 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 13 Oct 2020 10:41:21 +0200 Subject: [PATCH 3/7] Add explicit option to install pygelf --- bootstrap.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 8c8591b70e..73710d98db 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -21,18 +21,17 @@ CMD() usage() { - echo "Usage: $0 [-h] [+docs]" + echo "Usage: $0 [-h] [+docs] [+pygelf]" echo "Bootstrap ReFrame by pulling all its dependencies" echo " -P EXEC Use EXEC as Python interpreter" - echo " -O Install optional Python packages using pip" echo " -h Print this help message and exit" echo " +docs Build also the documentation" + echo " +pygelf Install also the pygelf Python package" } -while getopts "hOP:" opt; do +while getopts "hP:" opt; do case $opt in - "O") optional="pygelf" ;; "P") python=$OPTARG ;; "h") usage && exit 0 ;; "?") usage && exit 0 ;; @@ -64,11 +63,11 @@ export PYTHONPATH=$(pwd)/external:$(pwd)/external/usr/lib/python$pyver/site-pack CMD $python -m pip install --no-cache-dir -q --upgrade pip --target=external/ CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade -if [ -n "$optional" ]; then - CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q $optional --target=external/ --upgrade +if [ x"$1" == x"+docs" -o x"$2" == x"+docs" ]; then + CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r docs/requirements.txt --target=external/ --upgrade + make -C docs PYTHON=$python fi -if [ x"$1" == x"+docs" ]; then - CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r docs/requirements.txt --target=external/ --upgrade - make -C docs +if [ x"$1" == x"+pygelf" -o x"$2" == x"+pygelf" ]; then + CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q pygelf --target=external/ --upgrade fi From 293e9b17a723e209d75ff7b8f07b2fab86c8f613 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Wed, 14 Oct 2020 09:32:28 +0200 Subject: [PATCH 4/7] Address PR comments --- bootstrap.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 73710d98db..956bf70286 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -43,8 +43,15 @@ if [ -z $python ]; then python=python3 fi -pyver=$($python -V | sed -n 's/Python \([0-9]\+\)\.\([0-9]\+\)\..*/\1.\2/p') +while [ -n "$1" ]; do + case "$1" in + "+docs") MAKEDOCS="true" && shift ;; + "+pygelf") PYGELF="true" && shift ;; + *) usage && exit 0 ;; + esac +done +pyver=$($python -V | sed -n 's/Python \([0-9]\+\)\.\([0-9]\+\)\..*/\1.\2/p') # Check if ensurepip is installed $python -m ensurepip --version &> /dev/null @@ -63,11 +70,11 @@ export PYTHONPATH=$(pwd)/external:$(pwd)/external/usr/lib/python$pyver/site-pack CMD $python -m pip install --no-cache-dir -q --upgrade pip --target=external/ CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade -if [ x"$1" == x"+docs" -o x"$2" == x"+docs" ]; then +if [ "$MAKEDOCS" == "true" ]; then CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r docs/requirements.txt --target=external/ --upgrade make -C docs PYTHON=$python fi -if [ x"$1" == x"+pygelf" -o x"$2" == x"+pygelf" ]; then +if [ "$PYGELF" == "true" ]; then CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q pygelf --target=external/ --upgrade fi From 295f2897960d36cca92d7fc753a886cff42046b3 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 15 Oct 2020 10:34:31 +0200 Subject: [PATCH 5/7] Pin optional pygelf version in requirements --- bootstrap.sh | 18 ++++++++++++------ requirements.txt | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 956bf70286..ae7d64007d 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -47,7 +47,7 @@ while [ -n "$1" ]; do case "$1" in "+docs") MAKEDOCS="true" && shift ;; "+pygelf") PYGELF="true" && shift ;; - *) usage && exit 0 ;; + *) usage && exit 1 ;; esac done @@ -68,13 +68,19 @@ export PATH=$(pwd)/external/usr/bin:$PATH export PYTHONPATH=$(pwd)/external:$(pwd)/external/usr/lib/python$pyver/site-packages:$PYTHONPATH CMD $python -m pip install --no-cache-dir -q --upgrade pip --target=external/ -CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade -if [ "$MAKEDOCS" == "true" ]; then +if [ -n "$PYGELF" ]; then + cp requirements.txt requirements_tmp.txt + sed -i -e 's/^#\.pygelf[[:space:]]\+//g' requirements.txt + CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade + mv requirements_tmp.txt requirements.txt +else + CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade +fi + +if [ -n "$MAKEDOCS" ]; then CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r docs/requirements.txt --target=external/ --upgrade make -C docs PYTHON=$python fi -if [ "$PYGELF" == "true" ]; then - CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q pygelf --target=external/ --upgrade -fi + diff --git a/requirements.txt b/requirements.txt index 3b4d79e1fa..75333aa2e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pytest>=5.0.0 coverage setuptools wcwidth +#.pygelf pygelf==0.3.6 From 59c91a51f2ab9279abd9df1e919bc1b8c183aa07 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 19 Oct 2020 14:00:20 +0200 Subject: [PATCH 6/7] Use mktemp to create temp requirements file --- bootstrap.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index ae7d64007d..c253a762b5 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -70,10 +70,10 @@ export PYTHONPATH=$(pwd)/external:$(pwd)/external/usr/lib/python$pyver/site-pack CMD $python -m pip install --no-cache-dir -q --upgrade pip --target=external/ if [ -n "$PYGELF" ]; then - cp requirements.txt requirements_tmp.txt - sed -i -e 's/^#\.pygelf[[:space:]]\+//g' requirements.txt - CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade - mv requirements_tmp.txt requirements.txt + tmp_requirements=$(mktemp requirements_XXX.txt) + sed -e 's/^#\.pygelf[[:space:]]\+//g' requirements.txt > $tmp_requirements + CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r $tmp_requirements --target=external/ --upgrade + rm $tmp_requirements else CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade fi From 4c31f8c2acb2d34e88e4794b2070d605a3a63138 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 19 Oct 2020 20:14:05 +0200 Subject: [PATCH 7/7] Make bootstrap script more portable --- bootstrap.sh | 17 ++++++++++------- requirements.txt | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index c253a762b5..cbcab84a4e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -19,6 +19,12 @@ CMD() echo -e "${BLUE}==>${NC}" ${YELLOW}$*${NC} && $* } +CMD_M() +{ + msg=$1 + shift && echo -e "${BLUE}==> [$msg]${NC}" ${YELLOW}$*${NC} && $* +} + usage() { echo "Usage: $0 [-h] [+docs] [+pygelf]" @@ -70,17 +76,14 @@ export PYTHONPATH=$(pwd)/external:$(pwd)/external/usr/lib/python$pyver/site-pack CMD $python -m pip install --no-cache-dir -q --upgrade pip --target=external/ if [ -n "$PYGELF" ]; then - tmp_requirements=$(mktemp requirements_XXX.txt) - sed -e 's/^#\.pygelf[[:space:]]\+//g' requirements.txt > $tmp_requirements - CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r $tmp_requirements --target=external/ --upgrade - rm $tmp_requirements + tmp_requirements=$(mktemp) + sed -e 's/^#+pygelf%//g' requirements.txt > $tmp_requirements + CMD_M +pygelf $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r $tmp_requirements --target=external/ --upgrade && rm $tmp_requirements else CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r requirements.txt --target=external/ --upgrade fi if [ -n "$MAKEDOCS" ]; then - CMD $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r docs/requirements.txt --target=external/ --upgrade + CMD_M +docs $python -m pip install --use-feature=2020-resolver --no-cache-dir -q -r docs/requirements.txt --target=external/ --upgrade make -C docs PYTHON=$python fi - - diff --git a/requirements.txt b/requirements.txt index 75333aa2e5..f911e6bf7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ pytest>=5.0.0 coverage setuptools wcwidth -#.pygelf pygelf==0.3.6 +#+pygelf%pygelf==0.3.6