ci container update for 3.4.1 (#19252) (#19271)

* ci container recipe updates * tweak to blosc2 paths * update ci tpl container * update to conduit 0.8.8 * refactor docker helper to run in gitbash on windows * mystery solving * more updates * round and round we go * update ci container * fix missing std limits include header * solve build_visit sickness * new container * fix qt runtime lib path for ci
visit-dav · Jan 31, 2024 · 910ffa8 · 910ffa8
1 parent 27a9ab1
commit 910ffa8
Show file tree

Hide file tree

Showing 14 changed files with 87 additions and 57 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -12,7 +12,7 @@
 # TO USE A NEW CONTAINER, UPDATE TAG NAME HERE AS PART OF YOUR PR!
 #####
 variables:
-  container_tag: visitdav/visit-ci-develop:2023-03-23-shafe8254
+  container_tag: visitdav/visit-ci-develop:2024-01-24-sha0d1435
 
 # only build merge target pr to develop
 trigger: none
@@ -126,13 +126,12 @@ stages:
            #################################
            # run test suite
            cd build/test
-           # find vtk, ospray and tbb
+           # find qt, vtk, and ospray
+           export QT_LIB_DIR=`ls -d ${TPLS_PATH}/qt/*/ci/lib/`
            export VTK_LIB_DIR=`ls -d ${TPLS_PATH}/vtk/*/ci/lib/`
            export OSPRAY_LIB_DIR=`ls -d ${TPLS_PATH}/ospray/*/ci/lib/`
-           # note: tbb has extra paths under lib
-           export TBB_LIB_DIR=`ls -d ${TPLS_PATH}/tbb/*/ci/lib/intel64/gcc4.7/`
            # add to ld_lib path (rpaths are missing?)
-           export LD_LIBRARY_PATH=${VTK_LIB_DIR}:${OSPRAY_LIB_DIR}:${TBB_LIB_DIR}
+           export LD_LIBRARY_PATH=${VTK_LIB_DIR}:${OSPRAY_LIB_DIR}:${QT_LIB_DIR}
            # run test suite on silo + blueprint tests
            export TESTS=../../src/test/tests/databases/silo.py
            export TESTS="${TESTS} ../../src/test/tests/databases/blueprint.py"

diff --git a/scripts/ci/docker/Dockerfile b/scripts/ci/docker/Dockerfile
@@ -1,30 +1,36 @@
 # Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
 # Project developers.  See the top-level LICENSE file for dates and other
 # details.  No copyright assignment is required to contribute to VisIt.
-FROM ubuntu:bionic
-MAINTAINER Cyrus Harrison <cyrush@llnl.gov>
+FROM ubuntu:22.04
 
 # fetch build env
 RUN apt-get update && apt-get install -y \
     git \
     git-lfs \
     wget \
     curl \
+    tar   \
     p7zip \
     unzip \
-    subversion \
     build-essential \
     gcc \
     g++ \
     gfortran \
     zlib1g-dev \
-    python \
+    python3 \
+    python3-dev \
+    python-is-python3 \
     libsm-dev \
     libice-dev \
     libssl-dev \
+    '^libxcb.*-dev' \
     libx11-xcb-dev \
     libxcb-dri2-0-dev \
     libxcb-xfixes0-dev \
+    libxrender-dev \
+    libxi-dev \
+    libxkbcommon-dev \
+    libxkbcommon-x11-dev \
     libffi-dev \
     xutils-dev \
     xorg-dev \
@@ -43,6 +49,7 @@ RUN apt-get update && apt-get install -y \
     locales \
  && rm -rf /var/lib/apt/lists/*
 
+
 RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
 ENV LANG en_US.utf8
 
@@ -53,24 +60,26 @@ RUN cd /usr/include && ln -s freetype2 freetype
 # USER ci
 # WORKDIR /home/ci
 
-
 # untar the current masonry source  (created as part of build_docker_visit_ci.py)
 COPY visit.masonry.docker.src.tar /
-RUN tar -xzf visit.masonry.docker.src.tar
+RUN tar -xf visit.masonry.docker.src.tar
+
 # untar the current build_visit source (created as part of build_docker_visit_ci.py)
-RUN mkdir -p /masonry/build-mb-develop-ci-smoke/visit/src/tools/dev/scripts/
-COPY visit.build_visit.docker.src.tar /masonry
-COPY masonry_docker_ci_cleanup.py /
-RUN cd /masonry/build-mb-develop-ci-smoke/visit/src/tools/dev/scripts/ && tar -xzf /masonry/visit.build_visit.docker.src.tar
+RUN mkdir -p /scripts/
+COPY visit.build_visit.docker.src.tar /scripts
+RUN cd /scripts/ && tar -xf visit.build_visit.docker.src.tar
 
 # ensure build_visit script is executable
-RUN cd /masonry/build-mb-develop-ci-smoke/visit/src/tools/dev/scripts/ && chmod u+x build_visit
+RUN cd /scripts/ && chmod u+x build_visit
 
 # call masonry to build tpls
-RUN cd masonry && python bootstrap_visit.py opts/mb-develop-ci-smoke.json
+RUN cd masonry && python3 bootstrap_visit.py opts/mb-develop-ci-smoke.json
 
+# copy cleanup helper
+COPY masonry_docker_ci_cleanup.py /
 # cleanup extract build dirs and obtain the config site file
 RUN python masonry_docker_ci_cleanup.py
+
 # change perms
 RUN chmod -R a+rX /masonry
 

diff --git a/scripts/ci/docker/build_docker_visit_ci.py b/scripts/ci/docker/build_docker_visit_ci.py
@@ -7,10 +7,10 @@
 #
 
 import os
-import sys
 import subprocess
 import shutil
 import datetime
+import tarfile
 
 from os.path import join as pjoin
 
@@ -77,14 +77,15 @@ def main():
     os.chdir("../../../src/tools/dev")
 
     # get current copy of masonry
-    cmd ='tar -czvf {0} --exclude "build-*" --exclude "*.pyc" masonry'
-    sexe(cmd.format(pjoin(orig_dir, "visit.masonry.docker.src.tar")))
+    with tarfile.open(pjoin(orig_dir, "visit.masonry.docker.src.tar"),'w') as fout_mas:
+        fout_mas.add("masonry",recursive=True)
 
     # get current copy of build_visit from this branch
     os.chdir("scripts")
-    cmd = 'tar -czvf {0} --exclude "build-*" --exclude "*.pyc" build_visit bv_support' 
-    sexe(cmd.format(pjoin(orig_dir, "visit.build_visit.docker.src.tar")))
-
+    with tarfile.open(pjoin(orig_dir, "visit.build_visit.docker.src.tar"),'w') as fout_bv:
+        fout_bv.add("build_visit")
+        fout_bv.add("bv_support",recursive=True)
+
     # change back to orig working dir
     os.chdir(orig_dir)
 

diff --git a/scripts/ci/docker/masonry_docker_ci_cleanup.py b/scripts/ci/docker/masonry_docker_ci_cleanup.py
@@ -26,7 +26,7 @@ def cleanup_tpl_build_dirs():
     paths = glob.glob( pjoin(tpl_build_dir(),"*") )
     for path in paths:
         if not check_to_keep(path):
-            print "[removing %s]" % path
+            print("[removing %s]" % path)
             if os.path.isdir(path):
                 shutil.rmtree(path)
             else:
@@ -35,7 +35,7 @@ def cleanup_tpl_build_dirs():
 def copy_config_site():
      cfg_site = glob.glob( pjoin(tpl_build_dir(),"*.cmake") )[0]
      dest = "/visit-ci-cfg.cmake"
-     print "[copying %s to %s]" % (cfg_site,dest)
+     print("[copying %s to %s]" % (cfg_site,dest))
      shutil.copyfile(cfg_site,dest)
 
 def main():

diff --git a/src/avt/Filters/avtSLIVRRayTracer.C b/src/avt/Filters/avtSLIVRRayTracer.C
@@ -9,6 +9,7 @@
 #include <avtSLIVRRayTracer.h>
 
 #include <vector>
+#include <limits>
 
 #include <visit-config.h>
 

diff --git a/src/tools/dev/masonry/bootstrap_visit.py b/src/tools/dev/masonry/bootstrap_visit.py
@@ -61,21 +61,6 @@ def load_opts(opts_json):
         opts["cert"] = ""
     return opts
 
-def visit_svn_path(path,svn_opts,branch=None,tag=None):
-    if svn_opts["mode"] == "anon":
-        res = "http://portal.nersc.gov/svn/visit/"
-    else:
-        nersc_uname = svn_opts["nersc_uname"]
-        res = "svn+ssh://%s@cori.nersc.gov/project/projectdirs/visit/svn/visit/"
-        res = res % nersc_uname
-    if not branch is None:
-        res = res + "branches/" + branch + "/" + path
-    elif not tag is None:
-        res = res + "tags/" + tag + "/" + path
-    else:
-        res = res + "trunk/" + path
-    return res
-
 def visit_git_path(git_opts):
     if git_opts["mode"] == "ssh":
         res = "ssh://git@github.com/visit-dav/visit.git"

diff --git a/src/tools/dev/masonry/opts/mb-develop-ci-smoke.json b/src/tools/dev/masonry/opts/mb-develop-ci-smoke.json
@@ -7,12 +7,12 @@
  "git": {"mode":"https",
          "depth":"1"},
  "build_visit":  { "cmake_ver": "3.24.3",
-                    "args":"--no-thirdparty --optional --no-moab --no-pidx --no-icet",
+                    "args":"--no-thirdparty --optional --no-moab --no-pidx --no-icet --no-pyside ",
                     "comments": "setup for ci testing, builds basics + optional tpls",
           "libs":["cmake",
                   "python",
                   "vtk",
-                  "qt",
+                  "qt6",
                   "qwt",
                   "llvm",
                   "mesagl",

diff --git a/src/tools/dev/scripts/bv_support/bv_adios2.sh b/src/tools/dev/scripts/bv_support/bv_adios2.sh
@@ -201,8 +201,20 @@ function build_adios2
         cfg_opts="${cfg_opts} -DCMAKE_CXX_FLAGS:STRING=\"${CXX_OPT_FLAGS}\""
         cfg_opts="${cfg_opts} -DADIOS2_USE_SST:BOOL=ON"
 
-        # Use Blosc2?
+        # Use Blosc2
         if [[ "$DO_BLOSC2" == "yes" ]] ; then
+            # find blosc2
+            BLOSC2_DIR="${VISITDIR}/blosc2/${BLOSC2_VERSION}/${VISITARCH}"
+            BLOSC2_INCLUDE_DIR="${BLOSC2_DIR}/include"
+            # note: lib dir can be `lib``, or `lib64` depending on the platform
+            if [[ -d "${BLOSC2_DIR}/lib64/" ]] ; then
+                BLOSC2_LIBRARY="${BLOSC2_DIR}/lib64/libblosc2.so"
+            fi
+
+            if [[ -d "${BLOSC2_DIR}/lib/" ]] ; then
+                BLOSC2_LIBRARY="${BLOSC2_DIR}/lib/libblosc2.so"
+            fi
+
             cfg_opts="${cfg_opts} -DADIOS2_USE_Blosc2:BOOL=ON"
             cfg_opts="${cfg_opts} -DBlosc2_DIR=${BLOSC2_DIR}"
             cfg_opts="${cfg_opts} -DBLOSC2_INCLUDE_DIR=${BLOSC2_INCLUDE_DIR}"

diff --git a/src/tools/dev/scripts/bv_support/bv_blosc2.sh b/src/tools/dev/scripts/bv_support/bv_blosc2.sh
@@ -52,11 +52,6 @@ function bv_blosc2_depends_on
 
 function build_blosc2
 {
-    # for use externally once installed
-    export BLOSC2_DIR="${VISITDIR}/blosc2/${BLOSC2_VERSION}/${VISITARCH}"
-    export BLOSC2_INCLUDE_DIR="${BLOSC2_DIR}/include"
-    export BLOSC2_LIBRARY="${BLOSC2_DIR}/lib64/libblosc2.so"
-
     #
     # Blosc2 uses CMake  -- make sure we have it built.
     #
@@ -123,6 +118,7 @@ function build_blosc2
         chmod -R ug+w,a+rX "$VISITDIR/blosc2"
         chgrp -R ${GROUP} "$VISITDIR/blosc2"
     fi
+
     cd "$START_DIR"
     info "Done with Blosc2"
     return 0

diff --git a/src/tools/dev/scripts/bv_support/bv_conduit.sh b/src/tools/dev/scripts/bv_support/bv_conduit.sh
@@ -38,12 +38,12 @@ function bv_conduit_depends_on
 
 function bv_conduit_info
 {
-    export CONDUIT_VERSION=${CONDUIT_VERSION:-"v0.8.7"}
+    export CONDUIT_VERSION=${CONDUIT_VERSION:-"v0.8.8"}
     export CONDUIT_FILE=${CONDUIT_FILE:-"conduit-${CONDUIT_VERSION}-src-with-blt.tar.gz"}
     export CONDUIT_COMPATIBILITY_VERSION=${CONDUIT_COMPATIBILITY_VERSION:-"v0.8.0"}
     export CONDUIT_BUILD_DIR=${CONDUIT_BUILD_DIR:-"conduit-${CONDUIT_VERSION}"}
-    export CONDUIT_MD5_CHECKSUM="a7747fedfa4f1452a8410d555e21eacf"
-    export CONDUIT_SHA256_CHECKSUM="f3bf44d860783f4e0d61517c5e280c88144af37414569f4cf86e2d29b3ba5293"
+    export CONDUIT_MD5_CHECKSUM="9a22a25b8d8a58d8c48982675b4ffc3e"
+    export CONDUIT_SHA256_CHECKSUM="99811e9c464b6f841f52fcd47e982ae47cbb01cba334cff43eabe13eea58c0df"
 }
 
 function bv_conduit_print

diff --git a/src/tools/dev/scripts/bv_support/bv_mfem.sh b/src/tools/dev/scripts/bv_support/bv_mfem.sh
@@ -189,6 +189,13 @@ function build_mfem
     if [[ "$DO_CONDUIT" == "yes" ]] ; then
         vopts="${vopts} -DMFEM_USE_CONDUIT=ON -DCONDUIT_DIR=${VISITDIR}/conduit/${CONDUIT_VERSION}/${VISITARCH}"
     fi
+
+    # when using conduit, mfem's cmake logic requires HDF5_DIR to find HDF5
+    # (NOTE: mfem could use CONDUIT_HDF5_DIR)
+    if [[ "$DO_HDF5" == "yes" ]] ; then
+        vopts="${vopts} -DHDF5_DIR=${VISITDIR}/hdf5/${HDF5_VERSION}/${VISITARCH}"
+    fi
+
     if [[ "$DO_FMS" == "yes" ]] ; then
         vopts="${vopts} -DMFEM_USE_FMS=ON -DFMS_DIR=${VISITDIR}/fms/${FMS_VERSION}/${VISITARCH}"
     else

diff --git a/src/tools/dev/scripts/bv_support/bv_openexr.sh b/src/tools/dev/scripts/bv_support/bv_openexr.sh
@@ -122,9 +122,19 @@ function build_ilmbase
          EXTRA_AC_FLAGS="ac_cv_build=aarch64-unknown-linux-gnu"
     fi
 
+    # Open EXR does not suport c++17, a few examples of errors:
+    #
+    #  ISO C++17 does not allow dynamic exception specifications
+    #  ISO C++17 does not allow ‘register’ storage class specifier [-Wregister]
+    #
+    # If c++17 s the compiler default it will fail to build
+    # Use --std=c++14 to avoid problems while building/
+    #
+    OPENEXR_EXTRA_CXX_FLAGS="--std=c++14"
+
     set -x
     ./configure ${OPTIONAL} CXX="$CXX_COMPILER" \
-                CC="$C_COMPILER" CFLAGS="$CFLAGS $C_OPT_FLAGS" CXXFLAGS="$CXXFLAGS $CXX_OPT_FLAGS" \
+                CC="$C_COMPILER" CFLAGS="$CFLAGS $C_OPT_FLAGS" CXXFLAGS="$CXXFLAGS $CXX_OPT_FLAGS $OPENEXR_EXTRA_CXX_FLAGS" \
                 $EXTRA_AC_FLAGS --prefix="$VISITDIR/openexr/$ILMBASE_VERSION/$VISITARCH" $DISABLE_BUILDTYPE
     set +x
     if [[ $? != 0 ]] ; then
@@ -196,8 +206,18 @@ function build_openexr
          EXTRA_AC_FLAGS="ac_cv_build=aarch64-unknown-linux-gnu"
     fi
 
+    # Open EXR does not suport c++17, a few examples of errors:
+    #
+    #  ISO C++17 does not allow dynamic exception specifications
+    #  ISO C++17 does not allow ‘register’ storage class specifier [-Wregister]
+    #
+    # If c++17 s the compiler default it will fail to build
+    # Use --std=c++14 to avoid problems while building/
+    #
+    OPENEXR_EXTRA_CXX_FLAGS="--std=c++14"
+
     ./configure ${OPTIONAL} CXX="$CXX_COMPILER" \
-                CC="$C_COMPILER" CFLAGS="$CFLAGS $C_OPT_FLAGS" CXXFLAGS="$CXXFLAGS $CXX_OPT_FLAGS" \
+                CC="$C_COMPILER" CFLAGS="$CFLAGS $C_OPT_FLAGS" CXXFLAGS="$CXXFLAGS $CXX_OPT_FLAGS $OPENEXR_EXTRA_CXX_FLAGS" \
                 $EXTRA_AC_FLAGS --prefix="$VISITDIR/openexr/$OPENEXR_VERSION/$VISITARCH" \
                 --with-ilmbase-prefix="$VISITDIR/openexr/$OPENEXR_VERSION/$VISITARCH" \
                 --with-pic --enable-imfexamples $DISABLE_BUILDTYPE

diff --git a/src/tools/dev/scripts/bv_support/bv_python.sh b/src/tools/dev/scripts/bv_support/bv_python.sh
@@ -446,7 +446,7 @@ function bv_python_info
     export EDITABLES_MD5_CHECKSUM=""
 
     export PLUGGY_URL=""
-    export PLUGGY_FILE="pluggy-1.2.0.tar.gz "
+    export PLUGGY_FILE="pluggy-1.2.0.tar.gz"
     export PLUGGY_BUILD_DIR="pluggy-1.2.0"
     export PLUGGY_MD5_CHECKSUM=""
 

diff --git a/src/tools/dev/scripts/bv_support/bv_qt6.sh b/src/tools/dev/scripts/bv_support/bv_qt6.sh
@@ -324,7 +324,7 @@ function build_qt6_base
         qt_cmake_flags="${qt_cmake_flags} -DOPENGL_gl_LIBRARY:STRING=${MESAGL_OPENGL_LIB}"
         qt_cmake_flags="${qt_cmake_flags} -DOPENGL_opengl_LIBRARY:STRING="
         qt_cmake_flags="${qt_cmake_flags} -DOPENGL_glu_LIBRARY:STRING=${MESAGL_GLU_LIB}"
-        qt_cmake_flags="${qt_cmake_flags} -DOPENGL_GL_PREFERENCE:STRING=LEGACY"
+        qt_cmake_flags="${qt_cmake_flags} -DOpenGL_GL_PREFERENCE:STRING=LEGACY"
     fi
     info "Configuring Qt6 base: . . . "
     set -x
@@ -344,15 +344,15 @@ function build_qt6_base
     # Build Qt. Config options above make sure we only build the libs & tools.
     #
     info "Building Qt6 base . . . "
-    $MAKE $MAKE_OPT_FLAGS
+    ${CMAKE_COMMAND} --build . --parallel $MAKE_OPT_FLAGS
 
     if [[ $? != 0 ]] ; then
         warn "Qt6 base build failed.  Giving up"
         return 1
     fi
 
     info "Installing Qt6  base . . . "
-    $MAKE install
+    ${CMAKE_COMMAND} --install .
 
     # Qt screws up permissions in some cases.  Try to fix that.
     chmod -R a+rX ${VISITDIR}/qt/${QT6_VERSION}