diff --git a/config/cscs-ci.py b/config/cscs-ci.py index e36392b115..0f8bc767ba 100644 --- a/config/cscs-ci.py +++ b/config/cscs-ci.py @@ -76,6 +76,7 @@ class ReframeSettings: 'access': ['--partition=cn-regression'], 'environs': ['builtin-gcc'], 'descr': 'Tsa compute nodes', + 'max_jobs': 10, 'resources': { '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], } diff --git a/config/cscs.py b/config/cscs.py index fe9349f8d5..c03c378742 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -300,6 +300,70 @@ class ReframeSettings: } }, + 'arolla': { + 'descr': 'Arolla MCH', + 'hostnames': [r'arolla-\w+\d+'], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': { + 'login': { + 'scheduler': 'local', + 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], + 'descr': 'Arolla login nodes', + }, + 'pn': { + 'scheduler': 'nativeslurm', + 'access': ['--partition=pn-regression'], + 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], + 'descr': 'Arolla post-processing nodes', + }, + 'cn': { + 'scheduler': 'nativeslurm', + 'access': ['--partition=cn-regression'], + 'environs': ['PrgEnv-gnu', 'PrgEnv-gnu-nompi', + 'PrgEnv-pgi', 'PrgEnv-pgi-nompi'], + 'descr': 'Arolla compute nodes', + 'resources': { + '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], + } + } + } + }, + + 'tsa': { + 'descr': 'Tsa MCH', + 'hostnames': [r'tsa-\w+\d+'], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/UES/reframe/resources', + 'partitions': { + 'login': { + 'scheduler': 'local', + 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], + 'descr': 'Tsa login nodes', + }, + 'pn': { + 'scheduler': 'nativeslurm', + 'access': ['--partition=pn-regression'], + 'environs': ['PrgEnv-pgi', 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu', 'PrgEnv-gnu-nompi'], + 'descr': 'Tsa post-processing nodes', + }, + 'cn': { + 'scheduler': 'nativeslurm', + 'access': ['--partition=cn-regression'], + 'environs': ['PrgEnv-gnu', 'PrgEnv-gnu-nompi', + 'PrgEnv-pgi', 'PrgEnv-pgi-nompi'], + 'descr': 'Tsa compute nodes', + 'resources': { + '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], + } + } + } + }, + 'generic': { 'descr': 'Generic example system', 'partitions': { @@ -378,41 +442,67 @@ class ReframeSettings: 'cxx': 'g++', 'ftn': 'gfortran', }, - 'PrgEnv-cray-c2sm': { - 'modules': ['c2sm-rcm/1.00.00-kesch', - 'c2sm/cray-env/base'], - }, - 'PrgEnv-cray-c2sm-gpu': { - 'modules': ['c2sm-rcm/1.00.00-kesch', - 'c2sm/cray-env/gpu'], + }, + + 'arolla': { + 'PrgEnv-pgi-nompi': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-pgi/19.9'], + 'cc': 'pgcc', + 'cxx': 'pgc++', + 'ftn': 'pgf90', }, - 'PrgEnv-pgi-c2sm': { - 'modules': ['c2sm-rcm/1.00.00-kesch', - 'c2sm/pgi-env/base'], + 'PrgEnv-pgi': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-pgi/19.9'], 'cc': 'mpicc', 'cxx': 'mpicxx', - 'ftn': 'mpif90', + 'ftn': 'mpifort', }, - 'PrgEnv-pgi-c2sm-gpu': { - 'modules': ['c2sm-rcm/1.00.00-kesch', - 'c2sm/pgi-env/gpu'], + 'PrgEnv-gnu': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu/19.2'], 'cc': 'mpicc', 'cxx': 'mpicxx', - 'ftn': 'mpif90', + 'ftn': 'mpifort', + }, + 'PrgEnv-gnu-nompi': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu/19.2'], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', + }, + }, + + 'tsa': { + 'PrgEnv-pgi-nompi': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-pgi/19.9'], + 'cc': 'pgcc', + 'cxx': 'pgc++', + 'ftn': 'pgf90', }, - 'PrgEnv-gnu-c2sm': { - 'modules': ['c2sm-rcm/1.00.00-kesch', - 'c2sm/gnu-env/base'], + 'PrgEnv-pgi': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-pgi/19.9'], 'cc': 'mpicc', 'cxx': 'mpicxx', - 'ftn': 'mpif90', + 'ftn': 'mpifort', }, - 'PrgEnv-gnu-c2sm-gpu': { - 'modules': ['c2sm-rcm/1.00.00-kesch', - 'c2sm/gnu-env/gpu'], + 'PrgEnv-gnu': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu/19.2'], 'cc': 'mpicc', 'cxx': 'mpicxx', - 'ftn': 'mpif90', + 'ftn': 'mpifort', + }, + 'PrgEnv-gnu-nompi': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu/19.2'], + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', }, }, diff --git a/cscs-checks/cuda/cuda_checks.py b/cscs-checks/cuda/cuda_checks.py index 83ec3ea1f4..4aa52827bb 100644 --- a/cscs-checks/cuda/cuda_checks.py +++ b/cscs-checks/cuda/cuda_checks.py @@ -5,16 +5,22 @@ class CudaCheck(rfm.RegressionTest): def __init__(self): - self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn', 'tiger:gpu'] + self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn', 'tiger:gpu', + 'arolla:cn', 'tsa:cn'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu'] if self.current_system.name == 'kesch': self.valid_prog_environs += ['PrgEnv-cray-nompi', 'PrgEnv-gnu-nompi'] + elif self.current_system.name in ['arolla', 'tsa']: + self.valid_prog_environs += ['PrgEnv-gnu-nompi'] self.sourcesdir = os.path.join(self.current_system.resourcesdir, 'CUDA', 'essentials') + if self.current_system.name == 'kesch': self.modules = ['cudatoolkit/8.0.61'] + elif self.current_system.name in ['arolla', 'tsa']: + self.modules = ['cuda/10.1.243'] else: self.modules = ['craype-accel-nvidia60'] @@ -23,6 +29,9 @@ def __init__(self): if self.current_system.name == 'kesch': self.exclusive_access = True self.nvidia_sm = '37' + elif self.current_system.name in ['arolla', 'tsa']: + self.exclusive_access = True + self.nvidia_sm = '70' self.maintainers = ['AJ', 'SK'] self.tags = {'production', 'craype', 'external-resources'} @@ -87,6 +96,9 @@ def __init__(self): self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu'] self.variables = {'G2G': '0'} self.num_gpus_per_node = 2 + elif self.current_system.name in ['arolla', 'tsa']: + self.valid_prog_environs = ['PrgEnv-gnu'] + self.num_gpus_per_node = 2 else: self.variables = {'CRAY_CUDA_MPS': '1'} diff --git a/cscs-checks/cuda/multi_gpu.py b/cscs-checks/cuda/multi_gpu.py index fb085135be..c7fe4510c7 100644 --- a/cscs-checks/cuda/multi_gpu.py +++ b/cscs-checks/cuda/multi_gpu.py @@ -8,10 +8,10 @@ @rfm.simple_test class GpuBandwidthCheck(rfm.RegressionTest): def __init__(self): - super().__init__() - self.valid_systems = ['kesch:cn', 'daint:gpu', 'dom:gpu', 'tiger:gpu'] + self.valid_systems = ['kesch:cn', 'daint:gpu', 'dom:gpu', 'tiger:gpu', + 'arolla:cn', 'tsa:cn'] self.valid_prog_environs = ['PrgEnv-gnu'] - if self.current_system.name == 'kesch': + if self.current_system.name in ['arolla', 'kesch', 'tsa']: self.valid_prog_environs = ['PrgEnv-gnu-nompi'] self.exclusive_access = True @@ -24,6 +24,8 @@ def __init__(self): nvidia_sm = '60' if self.current_system.name == 'kesch': nvidia_sm = '37' + elif self.current_system.name in ['arolla', 'tsa']: + nvidia_sm = '70' self.build_system.cxxflags = ['-I.', '-m64', '-arch=sm_%s' % nvidia_sm] self.sourcepath = 'bandwidthtestflex.cu' @@ -41,8 +43,11 @@ def __init__(self): if self.current_system.name in ['daint', 'dom', 'tiger']: self.modules = ['craype-accel-nvidia60'] self.num_gpus_per_node = 1 - else: - self.modules = ['craype-accel-nvidia35'] + elif self.current_system.name == 'kesch': + self.modules = ['cudatoolkit/8.0.61'] + self.num_gpus_per_node = 8 + elif self.current_system.name in ['arolla', 'tsa']: + self.modules = ['cuda/10.1.243'] self.num_gpus_per_node = 8 # perf_patterns and reference will be set by the sanity check function @@ -50,6 +55,11 @@ def __init__(self): self.perf_patterns = {} self.reference = {} self.__bwref = { + # FIXME: reference values for Arolla and Tsa need to be updated + # (sanity check fails if they are not defined) + 'arolla:cn:h2d': (7583, -0.1, None, 'MB/s'), + 'arolla:cn:d2h': (7584, -0.1, None, 'MB/s'), + 'arolla:cn:d2d': (137408, -0.1, None, 'MB/s'), 'daint:gpu:h2d': (11881, -0.1, None, 'MB/s'), 'daint:gpu:d2h': (12571, -0.1, None, 'MB/s'), 'daint:gpu:d2d': (499000, -0.1, None, 'MB/s'), @@ -62,6 +72,9 @@ def __init__(self): 'tiger:gpu:h2d': (0, None, None, 'MB/s'), 'tiger:gpu:d2h': (0, None, None, 'MB/s'), 'tiger:gpu:d2d': (0, None, None, 'MB/s'), + 'tsa:cn:h2d': (7583, -0.1, None, 'MB/s'), + 'tsa:cn:d2h': (7584, -0.1, None, 'MB/s'), + 'tsa:cn:d2d': (137408, -0.1, None, 'MB/s'), } self.tags = {'diagnostic', 'benchmark', 'mch', 'craype', 'external-resources'} diff --git a/cscs-checks/libraries/io/netcdf_compile_run.py b/cscs-checks/libraries/io/netcdf_compile_run.py index a6ded7da52..bb82d93d91 100644 --- a/cscs-checks/libraries/io/netcdf_compile_run.py +++ b/cscs-checks/libraries/io/netcdf_compile_run.py @@ -18,7 +18,8 @@ def __init__(self, lang, linkage): self.linkage = linkage self.descr = lang_names[lang] + ' NetCDF ' + linkage.capitalize() self.valid_systems = ['daint:gpu', 'daint:mc', - 'dom:gpu', 'dom:mc', 'kesch:cn', 'tiger:gpu'] + 'dom:gpu', 'dom:mc', 'kesch:cn', 'tiger:gpu', + 'arolla:cn', 'tsa:cn'] if self.current_system.name in ['daint', 'dom', 'tiger']: self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi'] @@ -30,6 +31,9 @@ def __init__(self, lang, linkage): if lang != 'f90': self.valid_prog_environs += ['PrgEnv-cray-nompi'] + elif self.current_system.name in ['arolla', 'tsa']: + self.exclusive_access = True + self.valid_prog_environs = ['PrgEnv-gnu-nompi', 'PrgEnv-pgi-nompi'] self.sourcesdir = os.path.join(self.current_system.resourcesdir, 'netcdf') @@ -81,6 +85,22 @@ def setflags(self): '-I$EBROOTNETCDFMINCPLUSPLUS/include', '-I$EBROOTNETCDFMINFORTRAN/include' ] + elif self.current_system.name in ['arolla', 'tsa']: + self.modules = ['netcdf', 'netcdf-c++', 'netcdf-fortran'] + self.build_system.cppflags = [ + '-I$EBROOTNETCDF/include', + '-I$EBROOTNETCDFMINCPLUSPLUS/include', + '-I$EBROOTNETCDFMINFORTRAN/include' + ] + self.build_system.ldflags = [ + '-L$EBROOTNETCDF/lib', + '-L$EBROOTNETCDFMINCPLUSPLUS/lib', + '-L$EBROOTNETCDFMINFORTRAN/lib', + '-L$EBROOTNETCDF/lib64', + '-L$EBROOTNETCDFMINCPLUSPLUS/lib64', + '-L$EBROOTNETCDFMINFORTRAN/lib64', + '-lnetcdf', '-lnetcdf_c++4', '-lnetcdff' + ] else: self.build_system.ldflags = ['-%s' % self.linkage] diff --git a/cscs-checks/mch/automatic_arrays_acc.py b/cscs-checks/mch/automatic_arrays_acc.py index 9c76e00c3b..d8e6b7f38e 100644 --- a/cscs-checks/mch/automatic_arrays_acc.py +++ b/cscs-checks/mch/automatic_arrays_acc.py @@ -5,9 +5,9 @@ @rfm.simple_test class AutomaticArraysCheck(rfm.RegressionTest): def __init__(self): - super().__init__() - self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn', 'tiger:gpu'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] + self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn', 'tiger:gpu', + 'arolla:cn', 'tsa:cn'] + self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-cce', 'PrgEnv-pgi'] if self.current_system.name in ['daint', 'dom', 'tiger']: self.modules = ['craype-accel-nvidia60'] elif self.current_system.name == 'kesch': @@ -19,6 +19,9 @@ def __init__(self): 'CRAY_ACCEL_TARGET': 'nvidia35', 'MV2_USE_CUDA': '1' } + elif self.current_system.name in ['arolla', 'tsa']: + self.exclusive_access = True + # This tets requires an MPI compiler, although it uses a single task self.num_tasks = 1 self.num_gpus_per_node = 1 @@ -31,7 +34,6 @@ def __init__(self): 'time': sn.extractsingle(r'Timing:\s+(?P