Skip to content

Commit

Permalink
Deploy spark@3.4.0 % gcc@10.2.0 into prod within expanse/0.17.3/cpu/b
Browse files Browse the repository at this point in the history
  • Loading branch information
@mkandes
mkandes committed Jul 11, 2023
1 parent 553657e commit bd28762
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 100 deletions.
191 changes: 92 additions & 99 deletions ...cc@10.2.0/spark@3.4.0.o22902088.exp-15-02 → ...cc@10.2.0/spark@3.4.0.o23942559.exp-15-56
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
1686323436 22902088 1511c7944d387d7e35502660c80f7377 /home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/spark@3.4.0.sh
1689034764 23942559 3cfed7250a53407ff910f4cf389cc7fc /cm/shared/apps/spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh

#!/usr/bin/env bash

#SBATCH --job-name=spark@3.4.0
#SBATCH --account=use300
##SBATCH --reservation=rocky8u7_testing
#SBATCH --reservation=root_73
#SBATCH --partition=ind-shared
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
Expand Down Expand Up @@ -82,132 +82,139 @@ Currently Loaded Modules:



CONDA_SHLVL=0
LD_LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
LS_COLORS=rs=0:di=38;5;33:ln=38;5;51:mh=00:pi=40;38;5;11:so=38;5;13:do=38;5;5:bd=48;5;232;38;5;11:cd=48;5;232;38;5;3:or=48;5;232;38;5;9:mi=01;05;37;41:su=48;5;196;38;5;15:sg=48;5;11;38;5;16:ca=48;5;196;38;5;226:tw=48;5;10;38;5;16:ow=48;5;10;38;5;21:st=48;5;21;38;5;15:ex=38;5;40:*.tar=38;5;9:*.tgz=38;5;9:*.arc=38;5;9:*.arj=38;5;9:*.taz=38;5;9:*.lha=38;5;9:*.lz4=38;5;9:*.lzh=38;5;9:*.lzma=38;5;9:*.tlz=38;5;9:*.txz=38;5;9:*.tzo=38;5;9:*.t7z=38;5;9:*.zip=38;5;9:*.z=38;5;9:*.dz=38;5;9:*.gz=38;5;9:*.lrz=38;5;9:*.lz=38;5;9:*.lzo=38;5;9:*.xz=38;5;9:*.zst=38;5;9:*.tzst=38;5;9:*.bz2=38;5;9:*.bz=38;5;9:*.tbz=38;5;9:*.tbz2=38;5;9:*.tz=38;5;9:*.deb=38;5;9:*.rpm=38;5;9:*.jar=38;5;9:*.war=38;5;9:*.ear=38;5;9:*.sar=38;5;9:*.rar=38;5;9:*.alz=38;5;9:*.ace=38;5;9:*.zoo=38;5;9:*.cpio=38;5;9:*.7z=38;5;9:*.rz=38;5;9:*.cab=38;5;9:*.wim=38;5;9:*.swm=38;5;9:*.dwm=38;5;9:*.esd=38;5;9:*.jpg=38;5;13:*.jpeg=38;5;13:*.mjpg=38;5;13:*.mjpeg=38;5;13:*.gif=38;5;13:*.bmp=38;5;13:*.pbm=38;5;13:*.pgm=38;5;13:*.ppm=38;5;13:*.tga=38;5;13:*.xbm=38;5;13:*.xpm=38;5;13:*.tif=38;5;13:*.tiff=38;5;13:*.png=38;5;13:*.svg=38;5;13:*.svgz=38;5;13:*.mng=38;5;13:*.pcx=38;5;13:*.mov=38;5;13:*.mpg=38;5;13:*.mpeg=38;5;13:*.m2v=38;5;13:*.mkv=38;5;13:*.webm=38;5;13:*.ogm=38;5;13:*.mp4=38;5;13:*.m4v=38;5;13:*.mp4v=38;5;13:*.vob=38;5;13:*.qt=38;5;13:*.nuv=38;5;13:*.wmv=38;5;13:*.asf=38;5;13:*.rm=38;5;13:*.rmvb=38;5;13:*.flc=38;5;13:*.avi=38;5;13:*.fli=38;5;13:*.flv=38;5;13:*.gl=38;5;13:*.dl=38;5;13:*.xcf=38;5;13:*.xwd=38;5;13:*.yuv=38;5;13:*.cgm=38;5;13:*.emf=38;5;13:*.ogv=38;5;13:*.ogx=38;5;13:*.aac=38;5;45:*.au=38;5;45:*.flac=38;5;45:*.m4a=38;5;45:*.mid=38;5;45:*.midi=38;5;45:*.mka=38;5;45:*.mp3=38;5;45:*.mpc=38;5;45:*.ogg=38;5;45:*.ra=38;5;45:*.wav=38;5;45:*.oga=38;5;45:*.opus=38;5;45:*.spx=38;5;45:*.xspf=38;5;45:
CONDA_EXE=/home/mkandes/miniconda3/bin/conda
LOCAL_SCRATCH_DIR=/scratch/mkandes/job_22902088
SRUN_DEBUG=3
SLURM_STEP_ID=0
LOCAL_SCRATCH_DIR=/scratch/spack_cpu/job_23942559
SLURM_NODEID=0
LOCAL_SPACK_TMPDIR=/tmp
SLURM_TASK_PID=482756
__LMOD_REF_COUNT_PATH=/cm/shared/apps/slurm/current/sbin:1;/cm/shared/apps/slurm/current/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/bin:1;/home/mkandes/miniconda3/condabin:1;/home/mkandes/.local/bin:1;/home/mkandes/bin:1;/usr/local/bin:1;/usr/bin:1;/usr/local/sbin:1;/usr/sbin:1
_ModuleTable002_=b2NhbC9tb2R1bGVmaWxlcyIsIi9jbS9zaGFyZWQvYXBwcy9hY2Nlc3MvbW9kdWxlZmlsZXMiLCIvZXRjL21vZHVsZWZpbGVzIiwiL3Vzci9zaGFyZS9tb2R1bGVmaWxlcyIsIi91c3Ivc2hhcmUvTW9kdWxlcy9tb2R1bGVmaWxlcyIsIi9jbS9zaGFyZWQvbW9kdWxlZmlsZXMiLH0sWyJzeXN0ZW1CYXNlTVBBVEgiXT0iL2NtL2xvY2FsL21vZHVsZWZpbGVzOi9jbS9zaGFyZWQvYXBwcy9hY2Nlc3MvbW9kdWxlZmlsZXM6L2NtL3NoYXJlZC9tb2R1bGVmaWxlczovZXRjL21vZHVsZWZpbGVzOi91c3Ivc2hhcmUvbW9kdWxlZmlsZXM6L3Vzci9zaGFyZS9Nb2R1bGVzL21vZHVsZWZpbGVzIix9
SSH_CONNECTION=208.58.214.56 42828 198.202.100.14 22
SLURM_TASK_PID=1647216
__LMOD_REF_COUNT_PATH=/cm/shared/apps/slurm/current/sbin:1;/cm/shared/apps/slurm/current/bin:1;/home/spack_cpu/.local/bin:2;/home/spack_cpu/bin:2;/usr/local/bin:1;/usr/bin:1;/usr/local/sbin:1;/usr/sbin:1
_ModuleTable002_=L21vZHVsZWZpbGVzIiwiL3Vzci9zaGFyZS9Nb2R1bGVzL21vZHVsZWZpbGVzIiwiL2NtL3NoYXJlZC9tb2R1bGVmaWxlcyIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvY20vbG9jYWwvbW9kdWxlZmlsZXM6L2NtL3NoYXJlZC9hcHBzL2FjY2Vzcy9tb2R1bGVmaWxlczovY20vc2hhcmVkL21vZHVsZWZpbGVzOi9ldGMvbW9kdWxlZmlsZXM6L3Vzci9zaGFyZS9tb2R1bGVmaWxlczovdXNyL3NoYXJlL01vZHVsZXMvbW9kdWxlZmlsZXMiLH0=
SSH_CONNECTION=198.202.100.14 44506 198.202.100.14 22
SPACK_PYTHON=/usr/bin/python3
SLURM_PRIO_PROCESS=0
SLURM_CPU_BIND_VERBOSE=quiet
LANG=en_US.UTF-8
SLURM_SUBMIT_DIR=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs
SLURM_SUBMIT_DIR=/cm/shared/apps/spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0
HISTCONTROL=ignoredups
SHARED_SPACK_INSTANCE_VERSION=b
SHARED_SPACK_ROOT=/cm/shared/apps/spack/0.17.3/cpu/b
HOSTNAME=exp-15-02
HOSTNAME=exp-15-56
LMOD_SYSTEM_DEFAULT_MODULES=DefaultModules
OLDPWD=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack
OLDPWD=/home/spack_cpu
__LMOD_REF_COUNT__LMFILES_=/cm/local/modulefiles/slurm/expanse/21.08.8:1
SLURM_CPUS_PER_TASK=16
SLURM_STEPID=0
SLURM_SRUN_COMM_HOST=10.21.0.20
SPACK_DEPENDENCIES=^openjdk@11.0.12_7/27cv2ps
SLURM_DISTRIBUTION=cyclic
ENVIRONMENT=BATCH
SPACK_VERSION=0.17.3
SLURM_PROCID=0
SPACK_INSTANCE_VERSION=b
SLURM_JOB_GID=300
SLURM_JOB_GID=11491
SPACK_VARIANTS=+hadoop
SPACK_INSTANCE_DIR=/cm/shared/apps/spack/0.17.3/cpu/b
__LMOD_REF_COUNT_LD_LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:1;/cm/shared/apps/slurm/current/lib64:1
SLURM_JOB_SCRIPT=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/spark@3.4.0.sh
SLURMD_NODENAME=exp-15-02
SLURM_CPU_BIND=quiet,mask_cpu:0x0000000000000000000000000000FFFF
SLURM_JOB_SCRIPT=/cm/shared/apps/spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh
SLURMD_NODENAME=exp-15-56
SLURM_TASKS_PER_NODE=1
S_COLORS=auto
_CE_M=
which_declare=declare -f
SPACK_COMPILER=gcc@10.2.0
XDG_SESSION_ID=179391
LOCAL_SPACK_ROOT=/home/mkandes/.spack/0.17.3/cpu/b/
SLURM_JOB_RESERVATION=root_73
XDG_SESSION_ID=388302
USER=spack_cpu
SLURM_NNODES=1
USER=mkandes
SHARED_SPACK_VERSION=0.17.3
SLURM_LAUNCH_NODE_IPADDR=10.21.0.20
SLURM_STEP_TASKS_PER_NODE=1
SLURM_NTASKS_PER_NODE=1
__LMOD_REF_COUNT_MODULEPATH=/home/mkandes/.spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64:2;/cm/local/modulefiles:1;/cm/shared/apps/access/modulefiles:1;/etc/modulefiles:1;/usr/share/modulefiles:1;/usr/share/Modules/modulefiles:1;/cm/shared/modulefiles:1
__LMOD_REF_COUNT_MODULEPATH=/cm/local/modulefiles:1;/cm/shared/apps/access/modulefiles:1;/etc/modulefiles:1;/usr/share/modulefiles:1;/usr/share/Modules/modulefiles:1;/cm/shared/modulefiles:1
__LMOD_REF_COUNT_LOADEDMODULES=slurm/expanse/21.08.8:1
PWD=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs
PWD=/cm/shared/apps/spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0
ENABLE_LMOD=1
SLURM_JOB_NODELIST=exp-15-02
HOME=/home/mkandes
SLURM_JOB_NODELIST=exp-15-56
HOME=/home/spack_cpu
SLURM_CLUSTER_NAME=expanse
CONDA_PYTHON_EXE=/home/mkandes/miniconda3/bin/python
LMOD_COLORIZE=yes
LOCAL_TIME=20230609T081036-0700
SLURM_NODELIST=exp-15-02
SSH_CLIENT=208.58.214.56 42828 22
LOCAL_TIME=20230710T171924-0700
SLURM_NODELIST=exp-15-56
SSH_CLIENT=198.202.100.14 44506 22
LMOD_VERSION=8.2.4
CPATH=/cm/shared/apps/slurm/current/include
SLURM_NTASKS=1
LMOD_SETTARG_CMD=:
SLURM_JOB_CPUS_PER_NODE=16
BASH_ENV=/usr/share/lmod/lmod/init/bash
SLURM_TOPOLOGY_ADDR=exp-15-02
_CE_CONDA=
SLURM_TOPOLOGY_ADDR=exp-15-56
SLURM_WORKING_CLUSTER=expanse:mgr1:6817:9472:109
SLURM_JOB_MD5SUM=1511c7944d387d7e35502660c80f7377 /home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/spark@3.4.0.sh
SLURM_JOB_MD5SUM=3cfed7250a53407ff910f4cf389cc7fc /cm/shared/apps/spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh
SPACK_PACKAGE=spark@3.4.0
__LMOD_REF_COUNT_LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:1;/cm/shared/apps/slurm/current/lib64:1
SLURM_STEP_NODELIST=exp-15-56
SLURM_JOB_NAME=spark@3.4.0
TMPDIR=/scratch/mkandes/job_22902088
SLURM_SRUN_COMM_PORT=38161
TMPDIR=/scratch/spack_cpu/job_23942559
LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
LMOD_sys=Linux
SLURM_JOBID=22902088
SLURM_JOBID=23942559
SYSTEM_NAME=expanse
_ModuleTable001_=X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPTcyMDAuMCxbImNfc2hvcnRUaW1lIl09MC43NjI2NjA5ODAyMjQ2MSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXtzbHVybT17WyJmbiJdPSIvY20vbG9jYWwvbW9kdWxlZmlsZXMvc2x1cm0vZXhwYW5zZS8yMS4wOC44IixbImZ1bGxOYW1lIl09InNsdXJtL2V4cGFuc2UvMjEuMDguOCIsWyJsb2FkT3JkZXIiXT0xLHByb3BUPXt9LFsic3RhY2tEZXB0aCJdPTAsWyJzdGF0dXMiXT0iYWN0aXZlIixbInVzZXJOYW1lIl09InNsdXJtIix9LH0sbXBhdGhBPXsiL2hvbWUvbWthbmRlcy8uc3BhY2svMC4xNy4zL2NwdS9iL3NoYXJlL3NwYWNrL2xtb2QvbGludXgtcm9ja3k4LXg4Nl82NCIsIi9jbS9s
_ModuleTable001_=X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXtzbHVybT17WyJmbiJdPSIvY20vbG9jYWwvbW9kdWxlZmlsZXMvc2x1cm0vZXhwYW5zZS8yMS4wOC44IixbImZ1bGxOYW1lIl09InNsdXJtL2V4cGFuc2UvMjEuMDguOCIsWyJsb2FkT3JkZXIiXT0xLHByb3BUPXt9LFsic3RhY2tEZXB0aCJdPTAsWyJzdGF0dXMiXT0iYWN0aXZlIixbInVzZXJOYW1lIl09InNsdXJtIix9LH0sbXBhdGhBPXsiL2NtL2xvY2FsL21vZHVsZWZpbGVzIiwiL2NtL3NoYXJlZC9hcHBzL2FjY2Vzcy9tb2R1bGVmaWxlcyIsIi9ldGMvbW9kdWxlZmlsZXMiLCIvdXNyL3NoYXJl
SLURM_CONF=/cm/shared/apps/slurm/var/etc/expanse/slurm.conf
LOADEDMODULES=slurm/expanse/21.08.8
__LMOD_REF_COUNT_MANPATH=/cm/shared/apps/slurm/current/man:2;/usr/share/lmod/lmod/share/man:1;/usr/local/share/man:1;/usr/share/man:1;/cm/local/apps/environment-modules/current/share/man:1
LOCAL_SPACK_NAMESPACE=mkandes
_ModuleTable003_=ay8xLjguMF8yNjUtYjAxIl09e1siZm4iXT0iL2NtL3NoYXJlZC9hcHBzL3NwYWNrLzAuMTcuMy9jcHUvYi9zaGFyZS9zcGFjay9sbW9kL2xpbnV4LXJvY2t5OC14ODZfNjQvZ2NjLzEwLjIuMC9vcGVuamRrLzEuOC4wXzI2NS1iMDEvY2dpeXhjYi5sdWEiLFsiZnVsbE5hbWUiXT0ib3Blbmpkay8xLjguMF8yNjUtYjAxL2NnaXl4Y2IiLFsibG9hZE9yZGVyIl09Nyxwcm9wVD17fSxbInN0YWNrRGVwdGgiXT0wLFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJvcGVuamRrLzEuOC4wXzI2NS1iMDEiLH0sc2RzYz17WyJmbiJdPSIvY20vc2hhcmVkL21vZHVsZWZpbGVzL3Nkc2MvMS4wLmx1YSIsWyJmdWxsTmFtZSJdPSJzZHNjLzEuMCIsWyJsb2FkT3JkZXIiXT00LHBy
__LMOD_REF_COUNT_MANPATH=/cm/shared/apps/slurm/current/man:3;/usr/share/lmod/lmod/share/man:1;/usr/local/share/man:2;/usr/share/man:2;/cm/local/apps/environment-modules/current/share/man:2
_ModuleTable003_=TmFtZSJdPSJzaGFyZWQiLH0sc2x1cm09e1siZm4iXT0iL2NtL2xvY2FsL21vZHVsZWZpbGVzL3NsdXJtL2V4cGFuc2UvMjEuMDguOCIsWyJmdWxsTmFtZSJdPSJzbHVybS9leHBhbnNlLzIxLjA4LjgiLFsibG9hZE9yZGVyIl09Myxwcm9wVD17fSxbInN0YWNrRGVwdGgiXT0yLFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJzbHVybSIsfSx9LG1wYXRoQT17Ii9jbS9zaGFyZWQvYXBwcy9zcGFjay8wLjE3LjMvY3B1L2Ivc2hhcmUvc3BhY2svbG1vZC9saW51eC1yb2NreTgteDg2XzY0L0NvcmUiLCIvY20vbG9jYWwvbW9kdWxlZmlsZXMiLCIvY20vc2hhcmVkL2FwcHMvYWNjZXNzL21vZHVsZWZpbGVzIiwiL2V0Yy9tb2R1bGVmaWxlcyIsIi91c3Ivc2hhcmUvbW9k
SLURM_NODE_ALIASES=(null)
SLURM_JOB_QOS=ind-shared-normal
LMOD_ROOT=/usr/share/lmod
SLURM_JOB_QOS=ind-shared-normal
SLURM_TOPOLOGY_ADDR_PATTERN=node
SSH_TTY=/dev/pts/199
MAIL=/var/spool/mail/mkandes
SSH_TTY=/dev/pts/583
MAIL=/var/spool/mail/spack_cpu
SLURM_CPUS_ON_NODE=16
LMOD_arch=x86_64
SLURM_JOB_NUM_NODES=1
__Init_Default_Modules=1
CMD_WLM_CLUSTER_NAME=expanse
SLURM_MEM_PER_NODE=32768
SPACK_ROOT=/cm/shared/apps/spack/0.17.3/cpu/b
SHELL=/bin/bash
TERM=xterm-256color
SLURM_JOB_UID=501506
SHELL=/bin/bash
_ModuleTable_Sz_=2
SLURM_JOB_UID=527834
__LMOD_REF_COUNT_CPATH=/cm/shared/apps/slurm/current/include:1
SLURM_JOB_PARTITION=ind-shared
SHARED_SPACK_INSTANCE_NAME=cpu
SLURM_PTY_WIN_ROW=24
SLURM_CPU_BIND_LIST=0x0000000000000000000000000000FFFF
SPACK_INSTANCE_NAME=cpu
SLURM_JOB_USER=mkandes
SLURM_JOB_USER=spack_cpu
SLURM_PTY_WIN_COL=80
SLURM_NPROCS=1
SHLVL=2
SLURM_SUBMIT_HOST=login02
UNIX_TIME=1686323436
SHLVL=3
SLURM_SUBMIT_HOST=exp-15-56
UNIX_TIME=1689034764
SLURM_JOB_ACCOUNT=use300
MANPATH=/cm/shared/apps/slurm/current/man:/usr/share/lmod/lmod/share/man:/cm/shared/apps/slurm/current/man:/home/mkandes/miniconda3/man:/usr/local/share/man:/usr/share/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/current/share/man:/cm/local/apps/environment-modules/current/share/man
SPACK_LD_LIBRARY_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/openjdk-1.8.0_265-b01-cgiyxcb6jjt2m35gpsay4gciaivyoyrv/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/lib64:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/lib:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
MANPATH=/cm/shared/apps/slurm/current/man:/usr/share/lmod/lmod/share/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/current/share/man:/cm/local/apps/environment-modules/current/share/man
LMOD_PREPEND_BLOCK=normal
MODULEPATH=/home/mkandes/.spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64:/cm/local/modulefiles:/cm/shared/apps/access/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/usr/share/Modules/modulefiles:/cm/shared/modulefiles
SLURM_STEP_LAUNCHER_PORT=38161
MODULEPATH=/cm/local/modulefiles:/cm/shared/apps/access/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/usr/share/Modules/modulefiles:/cm/shared/modulefiles
SLURM_PTY_PORT=38987
SLURM_GTIDS=0
LOGNAME=mkandes
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/501506/bus
XDG_RUNTIME_DIR=/run/user/501506
LOGNAME=spack_cpu
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/527834/bus
XDG_RUNTIME_DIR=/run/user/527834
MODULEPATH_ROOT=/usr/share/modulefiles
PATH=/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/cm/shared/apps/spack/0.17.3/cpu/b/bin:/home/mkandes/miniconda3/condabin:/home/mkandes/.local/bin:/home/mkandes/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin
SLURM_JOB_ID=22902088
PATH=/home/spack_cpu/.local/bin:/home/spack_cpu/bin:/cm/shared/apps/spack/0.17.3/cpu/b/bin:/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/home/spack_cpu/.local/bin:/home/spack_cpu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin
SLURM_JOB_ID=23942559
_LMFILES_=/cm/local/modulefiles/slurm/expanse/21.08.8
SLURM_CPU_BIND_TYPE=mask_cpu:
DEBUGINFOD_URLS=https://debuginfod.centos.org/
SLURM_STEP_NUM_TASKS=1
SCHEDULER_MODULE=slurm
MODULESHOME=/usr/share/lmod/lmod
LMOD_SETTARG_FULL_SUPPORT=no
HISTSIZE=1000
HISTSIZE=-1
LMOD_PKG=/usr/share/lmod/lmod
SLURM_STEP_NUM_NODES=1
SPACK_SPEC=spark@3.4.0 % gcc@10.2.0 +hadoop ^openjdk@11.0.12_7/27cv2ps
LMOD_CMD=/usr/share/lmod/lmod/libexec/lmod
SLURM_LOCALID=0
Expand Down Expand Up @@ -387,13 +394,12 @@ compilers:
extra_rpaths: []
config:
install_tree:
root: /home/mkandes/.spack/0.17.3/cpu/b//opt/spack
root: $spack/opt/spack
projections:
all: ${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}
template_dirs:
- /cm/shared/apps/spack/0.17.3/cpu/b/share/spack/templates

- $spack/share/spack/templates

# Temporary locations Spack can try to use for builds.
#
# Recommended options are given below.
Expand All @@ -419,41 +425,36 @@ config:
# The build stage can be purged with `spack clean --stage` and
# `spack clean -a`, so it is important that the specified directory uniquely
# identifies Spack staging to avoid accidentally wiping out non-Spack work.
module_roots:
tcl: /home/mkandes/.spack/0.17.3/cpu/b//share/spack/modules
lmod: /home/mkandes/.spack/0.17.3/cpu/b//share/spack/lmod
build_stage:
- /home/mkandes/.spack/0.17.3/cpu/b//var/spack/stage
- /tmp/mkandes/spack-stage
- $tempdir/$user/spack-stage
- $user_cache_path/stage
# - $spack/var/spack/stage

# Directory in which to run tests and store test results.
# Tests will be stored in directories named by date/time and package
# name/hash.
source_cache: /home/mkandes/.spack/0.17.3/cpu/b//var/spack/cache
misc_cache: ~/.spack/cache
test_stage: $user_cache_path/test
source_cache: $spack/var/spack/cache
misc_cache: $user_cache_path/cache
connect_timeout: 10
verify_ssl: true
suppress_gpg_warnings: false
install_missing_compilers: false
checksum: true
deprecated: false
dirty: false
build_language: C
locks: true
build_jobs: 1
url_fetch_method: urllib
ccache: false
concretizer: clingo
db_lock_timeout: 3
package_lock_timeout: null
shared_linking: rpath
allow_sgid: true
test_stage: $user_cache_path/test
deprecated: false
url_fetch_method: urllib
concretizer: clingo
terminal_title: false
debug: false
build_jobs: 16
mirrors:
spack-public: https://mirror.spack.io
modules:
Expand Down Expand Up @@ -606,44 +607,36 @@ packages:
read: world
write: user
repos:
- /home/mkandes/.spack/0.17.3/cpu/b//var/spack/repos/mkandes
- $spack/var/spack/repos/sdsc
- $spack/var/spack/repos/builtin
upstreams:
spack-instance-1:
install_tree: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack
upstreams: {}
Input spec
--------------------------------
[ ] .spark@3.4.0%gcc@10.2.0+hadoop
[ ] ^builtin.openjdk@11.0.12_7%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2

Concretized
--------------------------------
po6mvtn [ ] mkandes.spark@3.4.0%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" +hadoop arch=linux-rocky8-zen2
sntnkcq [ ] sdsc.spark@3.4.0%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" +hadoop arch=linux-rocky8-zen2
bfyu354 [b r ] ^builtin.hadoop@3.3.0%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2
27cv2ps [b r ] ^builtin.openjdk@11.0.12_7%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2

real 12.53
user 10.94
sys 0.31
real 9.82
user 8.28
sys 0.26
[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/openjdk-11.0.12_7-27cv2psb6xthjbwmjlx53otdbjxch6wg
==> Installing hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f
==> No binary for hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f found: installing from source
==> Using cached archive: /home/mkandes/.spack/0.17.3/cpu/b/var/spack/cache/_source-cache/archive/ea/ea1a0f0afcdfb9b6b9d261cdce5a99023d7e8f72d26409e87f69bda65c663688.tar.gz
==> No patches needed for hadoop
==> hadoop: Executing phase: 'install'
==> hadoop: Successfully installed hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f
Fetch: 1.00s. Build: 33m 55.12s. Total: 33m 56.12s.
[+] /home/mkandes/.spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f
==> Installing spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2
==> No binary for spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2 found: installing from source
==> Using cached archive: /home/mkandes/.spack/0.17.3/cpu/b/var/spack/cache/_source-cache/archive/6c/6c0366139dab03f9bfb788b27918e7e3d411adcfa5631de75ac91afac87fcf5f.tgz
[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f
==> Installing spark-3.4.0-sntnkcqkor2zmovkogcjf5qivzxdul5v
==> No binary for spark-3.4.0-sntnkcqkor2zmovkogcjf5qivzxdul5v found: installing from source
==> Warning: Expected user 527834 to own /scratch/spack_cpu, but it is owned by 0
==> Fetching https://archive.apache.org/dist/spark/spark-3.4.0/spark-3.4.0-bin-without-hadoop.tgz
==> No patches needed for spark
==> spark: Executing phase: 'install'
==> spark: Successfully installed spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2
Fetch: 0.57s. Build: 1m 43.23s. Total: 1m 43.80s.
[+] /home/mkandes/.spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2
real 2787.18
user 24.57
sys 13.98
==> spark: Successfully installed spark-3.4.0-sntnkcqkor2zmovkogcjf5qivzxdul5v
Fetch: 42.42s. Build: 2.84s. Total: 45.26s.
[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/spark-3.4.0-sntnkcqkor2zmovkogcjf5qivzxdul5v
real 55.59
user 11.73
sys 2.01
==> Regenerating lmod module files
==> OpenFOAM bashrc env: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/aocc-3.2.0/openfoam-2106-jz42us227mirxrhqjvojlaut2giuh74j/etc/bashrc
Loading

0 comments on commit bd28762

Please sign in to comment.