Casper site config updates and documentation updates

climbfuji · Sep 27, 2023 · 2f7a928 · 2f7a928
1 parent d609bef
commit 2f7a928
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 33 deletions.
diff --git a/configs/sites/casper/compilers.yaml b/configs/sites/casper/compilers.yaml
@@ -13,9 +13,9 @@ compilers::
       - intel/19.1.1
       environment:
         prepend_path:
-          PATH: '/glade/u/apps/dav/opt/gnu/9.1.0/bin'
-          CPATH: '/glade/u/apps/dav/opt/gnu/9.1.0/include'
-          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/opt/intel/2020u1/compilers_and_libraries_2020.1.217/linux/mpi/intel64/libfabric/lib:/glade/u/apps/dav/opt/gnu/9.1.0/lib64'
+          PATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/bin'
+          CPATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/include'
+          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/opt/intel/2020u1/compilers_and_libraries_2020.1.217/linux/mpi/intel64/libfabric/lib:/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/lib64'
         set:
           INTEL_LICENSE_FILE: '28518@128.117.177.41'
           LM_LICENSE_FILE: '28518@128.117.177.41'

diff --git a/configs/sites/casper/modules.yaml b/configs/sites/casper/modules.yaml
@@ -5,3 +5,5 @@ modules:
     lmod:
       exclude:
       - ecflow
+      include:
+      - python
diff --git a/configs/sites/casper/packages.yaml b/configs/sites/casper/packages.yaml
@@ -13,20 +13,6 @@ packages:
       prefix: /glade/u/apps/opt/intel/2020u1
       modules:
       - impi/2019.7.217
-  python:
-    buildable: False
-    externals:
-    - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib
-      prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12
-      modules:
-      - miniconda/3.9.12
-  py-pip:
-    buildable: False
-    externals:
-    - spec: py-pip@21.2.4
-      prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12
-      modules:
-      - miniconda/3.9.12
 
 ### All other external packages listed alphabetically
   autoconf:
@@ -45,10 +31,10 @@ packages:
     externals:
     - spec: binutils@2.27.43
       prefix: /usr
-  bison:
-    externals:
-    - spec: bison@3.0.4
-      prefix: /usr
+  #bison:
+  #  externals:
+  #  - spec: bison@3.0.4
+  #    prefix: /usr
   bzip2:
     externals:
     - spec: bzip2@1.0.6
@@ -84,7 +70,7 @@ packages:
     buildable: False
     externals:
     - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /glade/work/jedipara/cheyenne/spack-stack/ecflow-5.8.4
+      prefix: /glade/work/epicufsrt/contrib/spack-stack/casper/ecflow-5.8.4
       modules:
       - ecflow/5.8.4
   # Don't use, leads to duplicate packages being built
@@ -163,7 +149,7 @@ packages:
     buildable: False
     externals:
     - spec: mysql@8.0.31
-      prefix: /glade/work/jedipara/cheyenne/spack-stack/mysql-8.0.31
+      prefix: /glade/work/epicufsrt/contrib/spack-stack/casper/mysql-8.0.31
       modules:
       - mysql/8.0.31
   ncurses:
@@ -195,7 +181,7 @@ packages:
   qt:
     externals:
     - spec: qt@5.15.2
-      prefix: /glade/work/jedipara/cheyenne/spack-stack/qt-5.15.2/5.15.2/gcc_64
+      prefix: /glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2
   rsync:
     externals:
     - spec: rsync@3.1.2
@@ -208,10 +194,6 @@ packages:
     externals:
     - spec: sed@4.2.2
       prefix: /usr
-  sqlite:
-    externals:
-    - spec: sqlite@3.8.10.2~fts~functions+rtree
-      prefix: /usr
   subversion:
     externals:
     - spec: subversion@1.7.14

diff --git a/doc/modulefile_templates/gcc b/doc/modulefile_templates/gcc
@@ -0,0 +1,23 @@
+#%Module1.0
+
+module-whatis "Provides a gcc-10.1.0 installation for use with spack."
+
+conflict gnu
+conflict gcc
+
+proc ModulesHelp { } {
+puts stderr "Provides a gcc-10.1.0 installation for use with spack."
+}
+
+
+# Set this value
+set GCC_PATH "/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0"
+
+prepend-path PATH "${GCC_PATH}/bin"
+prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib"
+prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib64"
+prepend-path LIBRARY_PATH "${GCC_PATH}/lib"
+prepend-path LIBRARY_PATH "${GCC_PATH}/lib64"
+prepend-path CPATH "${GCC_PATH}/include"
+prepend-path CMAKE_PREFIX_PATH "${GCC_PATH}"
+prepend-path MANPATH "${GCC_PATH}/share/man"
diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst
@@ -408,7 +408,67 @@ mysql
 NCAR-Wyoming Casper
 ------------------------------
 
-Casper is co-located with Cheyenne and shares the parallel filesystem ``/glade`` and more with it. It is, however, a different operating system with a somewhat different software stack. spack-stack was installed on Casper after it was installed on Cheyenne, and prerequisites from Cheyenne were reused where possible (``miniconda``, ``qt``, ``ecflow``, ``mysql``). See below for information on how to install these packages.
+On Casper, there are problems with newer versions of the Intel compiler/MPI library when trying to run MPI jobs with just one task (``mpiexec -np 1``) - for JEDI, job hangs forever in a particular MPI communication call in oops. This is why an older version Intel 19 is used here.
+
+gcc
+   CISL does not provide a newer GNU compiler for Casper, and the Cheyenne compiler has problems with missing symbols. Build ``gcc`` etc. as follows and create a module file from template ``gcc`` in ``/glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles``.
+
+.. code-block:: console
+
+   module purge
+   mkdir -p /glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/src
+   cd /glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/src
+   wget http://ftp.mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-10.1.0/gcc-10.1.0.tar.gz
+   tar -xvzf gcc-10.1.0.tar.gz
+   cd gcc-10.1.0/
+   ./contrib/download_prerequisites
+   ./configure \
+       --prefix=/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0 \
+       --disable-multilib --enable-languages=c,c++ 2>&1 | tee log.config
+   make -j8 2>&1 | tee log.make
+   make install 2>&1 | tee log.install
+   # create modulefile
+
+qt (qt@5)
+   The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. Follow these instructions to build ``qt@5.15.2`` using ``gcc@10.10.0``. See also https://wiki.qt.io/Building_Qt_5_from_Git#Getting_the_source_code for building qt from source.
+
+.. code-block:: console
+
+   module purge
+   export LMOD_TMOD_FIND_FIRST=yes
+   module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles
+   module load gcc/10.1.0
+   mkdir -p /glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2/src
+   cd /glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2/src
+   git clone https://code.qt.io/qt/qt5.git
+   cd qt5/
+   git checkout 5.12
+   perl init-repository 2>&1 | tee log.init-repository
+   cd ..
+   mkdir qt5-build
+   cd qt5-build
+   ../qt5/configure -opensource -nomake examples -nomake tests \
+       -prefix "/glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2" 2>&1 | tee log.config
+   make -j4 2>&1 | tee log.make
+   make install 2>&1 | tee log.install
+   # If errors occur during the installation of qtlocation, ignore. This is one of the last steps
+   # and not needed for ecflow (consider not building this module in the first place ... todo).
+
+ecflow
+  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5`, and loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>`.
+
+.. code-block:: console
+
+   module purge
+   export LMOD_TMOD_FIND_FIRST=yes
+   module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles
+   module load gnu/10.1.0
+   module load python/3.7.9
+   module load qt/5.15.2
+   module load cmake/3.18.2
+
+mysql
+  ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/glade/work/epicufsrt/contrib/spack-stack/casper/mysql-8.0.31``.
 
 .. _MaintainersSection_Cheyenne:
 

diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst
@@ -453,13 +453,11 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
 
 .. code-block:: console
 
-   spack external find --scope system # use '--exclude' for troublesome packages like bison@:3.3, openssl@1.1.1, and cmake 3.20 and earlier
+   spack external find --scope system --exclude bison --exclude cmake --exclude curl
    spack external find --scope system perl
    spack external find --scope system wget
    spack external find --scope system mysql
    spack external find --scope system texlive
-   # On Ubuntu (but not on Red Hat):
-   spack external find --scope system curl
 
 5. Find compilers, add to site config's ``compilers.yaml``
 

diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
@@ -311,7 +311,7 @@ The following is required for building new spack environments and for using spac
    module purge
    export LMOD_TMOD_FIND_FIRST=yes
    module use /glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc
-   module load miniconda/3.9.12
+   module load python/3.7.9
    module load ecflow/5.8.4
    module load mysql/8.0.31