From fba97866de74e7daadb2b38b56c1a471f7585eda Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Fri, 2 Apr 2021 16:24:48 +0200 Subject: [PATCH 1/2] Pull base images for CI instead of building from scratch --- ci-scripts/dockerfiles/Lmod.dockerfile | 33 +--------------- ci-scripts/dockerfiles/Lmod77.dockerfile | 39 +------------------ ci-scripts/dockerfiles/Lmod77_base.dockerfile | 39 +++++++++++++++++++ ci-scripts/dockerfiles/Lmod_base.dockerfile | 33 ++++++++++++++++ ci-scripts/dockerfiles/Tmod32.dockerfile | 20 +--------- ci-scripts/dockerfiles/Tmod32_base.dockerfile | 15 +++++++ ci-scripts/dockerfiles/Tmod4.dockerfile | 35 +---------------- ci-scripts/dockerfiles/Tmod4_base.dockerfile | 33 ++++++++++++++++ 8 files changed, 124 insertions(+), 123 deletions(-) create mode 100644 ci-scripts/dockerfiles/Lmod77_base.dockerfile create mode 100644 ci-scripts/dockerfiles/Lmod_base.dockerfile create mode 100644 ci-scripts/dockerfiles/Tmod32_base.dockerfile create mode 100644 ci-scripts/dockerfiles/Tmod4_base.dockerfile diff --git a/ci-scripts/dockerfiles/Lmod.dockerfile b/ci-scripts/dockerfiles/Lmod.dockerfile index 534684555f..b7dc75017c 100644 --- a/ci-scripts/dockerfiles/Lmod.dockerfile +++ b/ci-scripts/dockerfiles/Lmod.dockerfile @@ -2,39 +2,8 @@ # Execute this from the top-level ReFrame source directory # -FROM ubuntu:20.04 -ENV TZ=Europe/Zurich -ENV DEBIAN_FRONTEND=noninteractive -ENV _LMOD_VER=8.4.12 - -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - -# ReFrame requirements -RUN \ - apt-get -y update && \ - apt-get -y install ca-certificates && \ - update-ca-certificates && \ - apt-get -y install gcc && \ - apt-get -y install make && \ - apt-get -y install git && \ - apt-get -y install python3 python3-pip - -# Required utilities -RUN apt-get -y install wget - -# Install Lmod -RUN \ - apt-get -y install lua5.3 lua-bit32:amd64 lua-posix:amd64 lua-posix-dev liblua5.3-0:amd64 liblua5.3-dev:amd64 tcl tcl-dev tcl8.6 tcl8.6-dev:amd64 libtcl8.6:amd64 && \ - wget -q https://github.com/TACC/Lmod/archive/${_LMOD_VER}.tar.gz -O lmod.tar.gz && \ - tar xzf lmod.tar.gz && \ - cd Lmod-${_LMOD_VER} && \ - ./configure && make install - -ENV BASH_ENV=/usr/local/lmod/lmod/init/profile - -USER rfmuser +FROM reframehpc/rfm-ci-base:lmod # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/ diff --git a/ci-scripts/dockerfiles/Lmod77.dockerfile b/ci-scripts/dockerfiles/Lmod77.dockerfile index 7c349fb17a..fab7992e19 100644 --- a/ci-scripts/dockerfiles/Lmod77.dockerfile +++ b/ci-scripts/dockerfiles/Lmod77.dockerfile @@ -2,45 +2,8 @@ # Execute this from the top-level ReFrame source directory # -# -# LMod versions prior to 8.2 emitted Python commands differently, so we use this -# Dockerfile to test the bindings of older versions -# - - -FROM ubuntu:20.04 - -ENV TZ=Europe/Zurich -ENV DEBIAN_FRONTEND=noninteractive -ENV _LMOD_VER=7.7 - -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - -# ReFrame requirements -RUN \ - apt-get -y update && \ - apt-get -y install ca-certificates && \ - update-ca-certificates && \ - apt-get -y install gcc && \ - apt-get -y install make && \ - apt-get -y install git && \ - apt-get -y install python3 python3-pip - -# Required utilities -RUN apt-get -y install wget - -# Install Lmod -RUN \ - apt-get -y install lua5.3 lua-bit32:amd64 lua-posix:amd64 lua-posix-dev liblua5.3-0:amd64 liblua5.3-dev:amd64 tcl tcl-dev tcl8.6 tcl8.6-dev:amd64 libtcl8.6:amd64 lua-filesystem:amd64 lua-filesystem-dev:amd64 && \ - wget -q https://github.com/TACC/Lmod/archive/${_LMOD_VER}.tar.gz -O lmod.tar.gz && \ - tar xzf lmod.tar.gz && \ - cd Lmod-${_LMOD_VER} && \ - ./configure && make install - -ENV BASH_ENV=/usr/local/lmod/lmod/init/profile -USER rfmuser +FROM reframehpc/rfm-ci-base:lmod77 # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/ diff --git a/ci-scripts/dockerfiles/Lmod77_base.dockerfile b/ci-scripts/dockerfiles/Lmod77_base.dockerfile new file mode 100644 index 0000000000..6662f12170 --- /dev/null +++ b/ci-scripts/dockerfiles/Lmod77_base.dockerfile @@ -0,0 +1,39 @@ +# +# LMod versions prior to 8.2 emitted Python commands differently, so we use this +# Dockerfile to test the bindings of older versions +# + + +FROM ubuntu:20.04 + +ENV TZ=Europe/Zurich +ENV DEBIAN_FRONTEND=noninteractive +ENV _LMOD_VER=7.7 + +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +# ReFrame requirements +RUN \ + apt-get -y update && \ + apt-get -y install ca-certificates && \ + update-ca-certificates && \ + apt-get -y install gcc make git python3 python3-pip + +# Required utilities +RUN apt-get -y install wget + +# Install Lmod +RUN \ + apt-get -y install lua5.3 lua-bit32:amd64 lua-posix:amd64 lua-posix-dev liblua5.3-0:amd64 liblua5.3-dev:amd64 tcl tcl-dev tcl8.6 tcl8.6-dev:amd64 libtcl8.6:amd64 lua-filesystem:amd64 lua-filesystem-dev:amd64 && \ + wget -q https://github.com/TACC/Lmod/archive/${_LMOD_VER}.tar.gz -O lmod.tar.gz && \ + tar xzf lmod.tar.gz && \ + cd Lmod-${_LMOD_VER} && \ + ./configure && make install && \ + cd .. && rm -rf lmod.tar.gz Lmod-${_LMOD_VER} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV BASH_ENV=/usr/local/lmod/lmod/init/profile + +USER rfmuser diff --git a/ci-scripts/dockerfiles/Lmod_base.dockerfile b/ci-scripts/dockerfiles/Lmod_base.dockerfile new file mode 100644 index 0000000000..948bb1af19 --- /dev/null +++ b/ci-scripts/dockerfiles/Lmod_base.dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:20.04 + +ENV TZ=Europe/Zurich +ENV DEBIAN_FRONTEND=noninteractive +ENV _LMOD_VER=8.4.12 + +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +# ReFrame requirements +RUN \ + apt-get -y update && \ + apt-get -y install ca-certificates && \ + update-ca-certificates && \ + apt-get -y install gcc make git python3 python3-pip + +# Required utilities +RUN apt-get -y install wget + +# Install Lmod +RUN \ + apt-get -y install lua5.3 lua-bit32:amd64 lua-posix:amd64 lua-posix-dev liblua5.3-0:amd64 liblua5.3-dev:amd64 tcl tcl-dev tcl8.6 tcl8.6-dev:amd64 libtcl8.6:amd64 && \ + wget -q https://github.com/TACC/Lmod/archive/${_LMOD_VER}.tar.gz -O lmod.tar.gz && \ + tar xzf lmod.tar.gz && \ + cd Lmod-${_LMOD_VER} && \ + ./configure && make install && \ + cd .. && rm -rf lmod.tar.gz Lmod-${_LMOD_VER} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV BASH_ENV=/usr/local/lmod/lmod/init/profile + +USER rfmuser diff --git a/ci-scripts/dockerfiles/Tmod32.dockerfile b/ci-scripts/dockerfiles/Tmod32.dockerfile index fdf6cd42d2..471f31c02b 100644 --- a/ci-scripts/dockerfiles/Tmod32.dockerfile +++ b/ci-scripts/dockerfiles/Tmod32.dockerfile @@ -2,25 +2,7 @@ # Execute this from the top-level ReFrame source directory # -FROM centos:7 - -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - -# ReFrame requirements -RUN \ - yum -y install gcc && \ - yum -y install make && \ - yum -y install git && \ - yum -y install python3 - -# # Required utilities -# RUN apt-get -y install wget - -# Install Tmod 3.2 -RUN yum -y install environment-modules - -USER rfmuser +FROM reframehpc/rfm-ci-base:tmod32 # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/ diff --git a/ci-scripts/dockerfiles/Tmod32_base.dockerfile b/ci-scripts/dockerfiles/Tmod32_base.dockerfile new file mode 100644 index 0000000000..447b15886d --- /dev/null +++ b/ci-scripts/dockerfiles/Tmod32_base.dockerfile @@ -0,0 +1,15 @@ +FROM centos:7 + +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +# ReFrame requirements +RUN \ + yum -y install gcc make git python3 + +# Install Tmod 3.2 +RUN yum -y install environment-modules && \ + yum clean all && \ + rm -rf /var/cache/yum + +USER rfmuser diff --git a/ci-scripts/dockerfiles/Tmod4.dockerfile b/ci-scripts/dockerfiles/Tmod4.dockerfile index 3313ee3465..649798b08b 100644 --- a/ci-scripts/dockerfiles/Tmod4.dockerfile +++ b/ci-scripts/dockerfiles/Tmod4.dockerfile @@ -2,40 +2,7 @@ # Execute this from the top-level ReFrame source directory # -FROM ubuntu:20.04 - -ENV TZ=Europe/Zurich -ENV DEBIAN_FRONTEND=noninteractive -ENV _TMOD_VER=4.6.0 - -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - -# ReFrame requirements -RUN \ - apt-get -y update && \ - apt-get -y install ca-certificates && \ - update-ca-certificates && \ - apt-get -y install gcc && \ - apt-get -y install make && \ - apt-get -y install git && \ - apt-get -y install python3 python3-pip - -# Required utilities -RUN apt-get -y install wget - -# Install Tmod4 -RUN \ - apt-get -y install autoconf && \ - apt-get -y install tcl-dev && \ - wget -q https://github.com/cea-hpc/modules/archive/v${_TMOD_VER}.tar.gz -O tmod.tar.gz && \ - tar xzf tmod.tar.gz && \ - cd modules-${_TMOD_VER} && \ - ./configure && make install - -ENV BASH_ENV=/usr/local/Modules/init/profile.sh - -USER rfmuser +FROM reframehpc/rfm-ci-base:tmod4 # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/ diff --git a/ci-scripts/dockerfiles/Tmod4_base.dockerfile b/ci-scripts/dockerfiles/Tmod4_base.dockerfile new file mode 100644 index 0000000000..86a0176d2c --- /dev/null +++ b/ci-scripts/dockerfiles/Tmod4_base.dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:20.04 + +ENV TZ=Europe/Zurich +ENV DEBIAN_FRONTEND=noninteractive +ENV _TMOD_VER=4.6.0 + +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +# ReFrame requirements +RUN \ + apt-get -y update && \ + apt-get -y install ca-certificates && \ + update-ca-certificates && \ + apt-get -y install gcc make git python3 python3-pip + +# Required utilities +RUN apt-get -y install wget + +# Install Tmod4 +RUN \ + apt-get -y install autoconf tcl-dev && \ + wget -q https://github.com/cea-hpc/modules/archive/v${_TMOD_VER}.tar.gz -O tmod.tar.gz && \ + tar xzf tmod.tar.gz && \ + cd modules-${_TMOD_VER} && \ + ./configure && make install && \ + cd .. && rm -rf tmod.tar.gz modules-${_TMOD_VER} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV BASH_ENV=/usr/local/Modules/init/profile.sh + +USER rfmuser From 53095ed2f6be969af118876a6a7fda967d02dd75 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 13 Apr 2021 14:53:36 +0200 Subject: [PATCH 2/2] Address PR comments --- .../{Lmod_base.dockerfile => Lmod-base.dockerfile} | 5 ----- ci-scripts/dockerfiles/Lmod.dockerfile | 5 +++++ .../{Lmod77_base.dockerfile => Lmod77-base.dockerfile} | 5 ----- ci-scripts/dockerfiles/Lmod77.dockerfile | 5 +++++ .../{Tmod32_base.dockerfile => Tmod32-base.dockerfile} | 5 ----- ci-scripts/dockerfiles/Tmod32.dockerfile | 5 +++++ .../{Tmod4_base.dockerfile => Tmod4-base.dockerfile} | 5 ----- ci-scripts/dockerfiles/Tmod4.dockerfile | 5 +++++ 8 files changed, 20 insertions(+), 20 deletions(-) rename ci-scripts/dockerfiles/{Lmod_base.dockerfile => Lmod-base.dockerfile} (92%) rename ci-scripts/dockerfiles/{Lmod77_base.dockerfile => Lmod77-base.dockerfile} (94%) rename ci-scripts/dockerfiles/{Tmod32_base.dockerfile => Tmod32-base.dockerfile} (75%) rename ci-scripts/dockerfiles/{Tmod4_base.dockerfile => Tmod4-base.dockerfile} (91%) diff --git a/ci-scripts/dockerfiles/Lmod_base.dockerfile b/ci-scripts/dockerfiles/Lmod-base.dockerfile similarity index 92% rename from ci-scripts/dockerfiles/Lmod_base.dockerfile rename to ci-scripts/dockerfiles/Lmod-base.dockerfile index 948bb1af19..6ead08e747 100644 --- a/ci-scripts/dockerfiles/Lmod_base.dockerfile +++ b/ci-scripts/dockerfiles/Lmod-base.dockerfile @@ -4,9 +4,6 @@ ENV TZ=Europe/Zurich ENV DEBIAN_FRONTEND=noninteractive ENV _LMOD_VER=8.4.12 -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - # ReFrame requirements RUN \ apt-get -y update && \ @@ -29,5 +26,3 @@ RUN \ rm -rf /var/lib/apt/lists/* ENV BASH_ENV=/usr/local/lmod/lmod/init/profile - -USER rfmuser diff --git a/ci-scripts/dockerfiles/Lmod.dockerfile b/ci-scripts/dockerfiles/Lmod.dockerfile index b7dc75017c..e96afe6841 100644 --- a/ci-scripts/dockerfiles/Lmod.dockerfile +++ b/ci-scripts/dockerfiles/Lmod.dockerfile @@ -5,6 +5,11 @@ FROM reframehpc/rfm-ci-base:lmod +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +USER rfmuser + # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/ diff --git a/ci-scripts/dockerfiles/Lmod77_base.dockerfile b/ci-scripts/dockerfiles/Lmod77-base.dockerfile similarity index 94% rename from ci-scripts/dockerfiles/Lmod77_base.dockerfile rename to ci-scripts/dockerfiles/Lmod77-base.dockerfile index 6662f12170..53238abab4 100644 --- a/ci-scripts/dockerfiles/Lmod77_base.dockerfile +++ b/ci-scripts/dockerfiles/Lmod77-base.dockerfile @@ -10,9 +10,6 @@ ENV TZ=Europe/Zurich ENV DEBIAN_FRONTEND=noninteractive ENV _LMOD_VER=7.7 -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - # ReFrame requirements RUN \ apt-get -y update && \ @@ -35,5 +32,3 @@ RUN \ rm -rf /var/lib/apt/lists/* ENV BASH_ENV=/usr/local/lmod/lmod/init/profile - -USER rfmuser diff --git a/ci-scripts/dockerfiles/Lmod77.dockerfile b/ci-scripts/dockerfiles/Lmod77.dockerfile index fab7992e19..24da98da33 100644 --- a/ci-scripts/dockerfiles/Lmod77.dockerfile +++ b/ci-scripts/dockerfiles/Lmod77.dockerfile @@ -5,6 +5,11 @@ FROM reframehpc/rfm-ci-base:lmod77 +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +USER rfmuser + # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/ diff --git a/ci-scripts/dockerfiles/Tmod32_base.dockerfile b/ci-scripts/dockerfiles/Tmod32-base.dockerfile similarity index 75% rename from ci-scripts/dockerfiles/Tmod32_base.dockerfile rename to ci-scripts/dockerfiles/Tmod32-base.dockerfile index 447b15886d..2b52cd78a0 100644 --- a/ci-scripts/dockerfiles/Tmod32_base.dockerfile +++ b/ci-scripts/dockerfiles/Tmod32-base.dockerfile @@ -1,8 +1,5 @@ FROM centos:7 -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - # ReFrame requirements RUN \ yum -y install gcc make git python3 @@ -11,5 +8,3 @@ RUN \ RUN yum -y install environment-modules && \ yum clean all && \ rm -rf /var/cache/yum - -USER rfmuser diff --git a/ci-scripts/dockerfiles/Tmod32.dockerfile b/ci-scripts/dockerfiles/Tmod32.dockerfile index 471f31c02b..5f1d97372b 100644 --- a/ci-scripts/dockerfiles/Tmod32.dockerfile +++ b/ci-scripts/dockerfiles/Tmod32.dockerfile @@ -4,6 +4,11 @@ FROM reframehpc/rfm-ci-base:tmod32 +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +USER rfmuser + # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/ diff --git a/ci-scripts/dockerfiles/Tmod4_base.dockerfile b/ci-scripts/dockerfiles/Tmod4-base.dockerfile similarity index 91% rename from ci-scripts/dockerfiles/Tmod4_base.dockerfile rename to ci-scripts/dockerfiles/Tmod4-base.dockerfile index 86a0176d2c..bdea0bb358 100644 --- a/ci-scripts/dockerfiles/Tmod4_base.dockerfile +++ b/ci-scripts/dockerfiles/Tmod4-base.dockerfile @@ -4,9 +4,6 @@ ENV TZ=Europe/Zurich ENV DEBIAN_FRONTEND=noninteractive ENV _TMOD_VER=4.6.0 -# ReFrame user -RUN useradd -ms /bin/bash rfmuser - # ReFrame requirements RUN \ apt-get -y update && \ @@ -29,5 +26,3 @@ RUN \ rm -rf /var/lib/apt/lists/* ENV BASH_ENV=/usr/local/Modules/init/profile.sh - -USER rfmuser diff --git a/ci-scripts/dockerfiles/Tmod4.dockerfile b/ci-scripts/dockerfiles/Tmod4.dockerfile index 649798b08b..085471a55f 100644 --- a/ci-scripts/dockerfiles/Tmod4.dockerfile +++ b/ci-scripts/dockerfiles/Tmod4.dockerfile @@ -4,6 +4,11 @@ FROM reframehpc/rfm-ci-base:tmod4 +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +USER rfmuser + # Install ReFrame from the current directory COPY --chown=rfmuser . /home/rfmuser/reframe/