Skip to content

[test] Adapt MCH regression tests to C2SM-RCM environments #513

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Nov 23, 2018
Merged

[test] Adapt MCH regression tests to C2SM-RCM environments #513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1ce6fb2
WIP: automatic_arrays
ajocksch Oct 12, 2018
0273418
Merge branch 'master' of https://github.com/ajocksch/reframe into che…
ajocksch Oct 30, 2018
706e2c7
WIP: c2sm formally introduced; segfaults
ajocksch Oct 30, 2018
b5b09ff
WIP: more green
ajocksch Nov 5, 2018
f3c9c65
all checks green
ajocksch Nov 6, 2018
adb04e5
incorporated changes requested
ajocksch Nov 21, 2018
ba6d0b1
Fine tune MCH tests and redefine C2SM environments
Nov 23, 2018
71ae1a4
Merge branch 'master' into checks/c2sm
vkarak Nov 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions config/cscs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,8 @@ class ReframeSettings:
'PrgEnv-pgi-c2sm',
'PrgEnv-gnu-c2sm',
'PrgEnv-cray-c2sm-gpu',
'PrgEnv-gnu-c2sm-gpu',
'PrgEnv-cray-c2sm-gpu'],
'PrgEnv-pgi-c2sm-gpu',
'PrgEnv-gnu-c2sm-gpu'],
'descr': 'Kesch compute nodes',
'resources': {
'_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'],
Expand Down Expand Up @@ -244,36 +244,42 @@ class ReframeSettings:
},
'PrgEnv-cray-c2sm': {
'type': 'ProgEnvironment',
'modules': ['env', 'c2sm/cray-env/base'],
'modules': ['c2sm-rcm/1.00.00-kesch',
'c2sm/cray-env/base'],
},
'PrgEnv-cray-c2sm-gpu': {
'type': 'ProgEnvironment',
'modules': ['env', 'c2sm/cray-env/gpu'],
'modules': ['c2sm-rcm/1.00.00-kesch',
'c2sm/cray-env/gpu'],
},
'PrgEnv-pgi-c2sm': {
'type': 'ProgEnvironment',
'modules': ['env', 'c2sm/pgi-env/base'],
'modules': ['c2sm-rcm/1.00.00-kesch',
'c2sm/pgi-env/base'],
'cc': 'mpicc',
'cxx': 'mpicxx',
'ftn': 'mpif90',
},
'PrgEnv-pgi-c2sm-gpu': {
'type': 'ProgEnvironment',
'modules': ['env', 'c2sm/pgi-env/gpu'],
'modules': ['c2sm-rcm/1.00.00-kesch',
'c2sm/pgi-env/gpu'],
'cc': 'mpicc',
'cxx': 'mpicxx',
'ftn': 'mpif90',
},
'PrgEnv-gnu-c2sm': {
'type': 'ProgEnvironment',
'modules': ['env', 'c2sm/gnu-env/base'],
'modules': ['c2sm-rcm/1.00.00-kesch',
'c2sm/gnu-env/base'],
'cc': 'mpicc',
'cxx': 'mpicxx',
'ftn': 'mpif90',
},
'PrgEnv-gnu-c2sm-gpu': {
'type': 'ProgEnvironment',
'modules': ['env', 'c2sm/gnu-env/gpu'],
'modules': ['c2sm-rcm/1.00.00-kesch',
'c2sm/gnu-env/gpu'],
'cc': 'mpicc',
'cxx': 'mpicxx',
'ftn': 'mpif90',
Expand Down
17 changes: 13 additions & 4 deletions cscs-checks/mch/automatic_arrays.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,14 @@ def __init__(self):
super().__init__()
self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi',
'PrgEnv-gnu']
'PrgEnv-gnu', 'PrgEnv-cray-c2sm-gpu',
'PrgEnv-pgi-c2sm-gpu']
if self.current_system.name in ['daint', 'dom']:
self.modules = ['craype-accel-nvidia60']
elif self.current_system.name in ['kesch']:
self.modules = ['craype-accel-nvidia35']
# FIXME: workaround -- the variable should not be needed since
# there is no GPUdirect in this check
self.variables = {'MV2_USE_CUDA': '1'}

# This tets requires an MPI compiler, although it uses a single task
Expand Down Expand Up @@ -50,14 +53,20 @@ def __init__(self):
self.tags = {'production'}

def setup(self, partition, environ, **job_opts):
if environ.name == 'PrgEnv-cray':
if environ.name.startswith('PrgEnv-cray'):
envname = 'PrgEnv-cray'
self.build_system.fflags += ['-hacc', '-hnoomp']
elif environ.name == 'PrgEnv-pgi':
elif environ.name.startswith('PrgEnv-pgi'):
envname = 'PrgEnv-pgi'
self.build_system.fflags += ['-acc']
if self.current_system.name == 'kesch':
self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0']
elif self.current_system.name in ['daint', 'dom']:
self.build_system.fflags += ['-ta=tesla,cc60', '-Mnorpath']
elif environ.name.startswith('PrgEnv-gnu'):
envname = 'PrgEnv-gnu'
else:
envname = environ.name

self.reference = self.arrays_reference[environ.name]
self.reference = self.arrays_reference[envname]
super().setup(partition, environ, **job_opts)
3 changes: 3 additions & 0 deletions cscs-checks/mch/g2g_meteoswiss_check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ def __init__(self, g2g):
self.descr = 'G2G Meteoswiss check with G2G=%s' % g2g
self.strict_check = False
self.valid_systems = ['kesch:cn']

# FIXME: temporary workaround until the mvapich module is fixed;
# 'PrgEnv-gnu-c2sm-gpu' will be added later
self.valid_prog_environs = ['PrgEnv-gnu']
self.modules = ['cmake', 'craype-accel-nvidia35']
self.pre_run = ["export EXECUTABLE=$(ls src/ | "
Expand Down
6 changes: 5 additions & 1 deletion cscs-checks/mch/gpu_direct_acc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ def __init__(self):
super().__init__()
self.descr = 'tests gpu-direct for Fortran OpenACC'
self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi']

# FIXME: temporary workaround until the mvapich module is fixed;
# 'PrgEnv-pgi-c2sm-gpu' will be added later
self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-cray-c2sm-gpu',
'PrgEnv-pgi']
if self.current_system.name in ['daint', 'dom']:
self.modules = ['craype-accel-nvidia60']
self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'}
Expand Down
2 changes: 2 additions & 0 deletions cscs-checks/mch/gpu_direct_cuda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ def __init__(self):
super().__init__()
self.descr = 'tests gpu-direct for CUDA'
self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
# FIXME: temporary workaround until the mvapich module is fixed
# 'PrgEnv-gnu-c2sm-gpu' will be added later
self.valid_prog_environs = ['PrgEnv-gnu']
self.sourcepath = 'gpu_direct_cuda.cu'
self.build_system = 'SingleSource'
Expand Down
47 changes: 33 additions & 14 deletions cscs-checks/mch/openacc_cuda_mpi_cppstd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,36 +10,45 @@ def __init__(self, withmpi):
self.name = 'OpenaccCudaCPP' + name_suffix
self.descr = 'test for OpenACC, CUDA, MPI, and C++'
self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi']
self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', 'PrgEnv-gnu']
self.build_system = 'Make'
self.build_system.fflags = ['-O2']
if self.current_system.name in ['daint', 'dom']:
self.modules = ['craype-accel-nvidia60']
self.variables = {
'MPICH_RDMA_ENABLED_CUDA': '1',
'CRAY_CUDA_MPS': '1'
}
self.num_tasks = 12
self.num_tasks_per_node = 12
self.num_gpus_per_node = 1
self.build_system.options = ['NVCC_FLAGS="-arch=compute_60"']
elif self.current_system.name in ['kesch']:
elif self.current_system.name == 'kesch':
self.modules = ['craype-accel-nvidia35']
self.variables = {
'MV2_USE_CUDA': '1',
'G2G': '1'
}
self.num_tasks = 8
self.num_tasks_per_node = 8
self.num_gpus_per_node = 8
self.build_system.options = ['NVCC_FLAGS="-arch=compute_37"']
# FIXME: temporary workaround until the mvapich module is fixed;
# 'PrgEnv-{pgi,gnu}-c2sm-gpu' will be added later
self.valid_prog_environs += ['PrgEnv-cray-c2sm-gpu']

if withmpi:
self.build_system.cppflags = ['-DUSE_MPI']
if self.current_system.name in ['daint', 'dom']:
self.variables = {
'MPICH_RDMA_ENABLED_CUDA': '1',
'CRAY_CUDA_MPS': '1'
}
elif self.current_system.name in ['kesch']:
self.variables = {
'MV2_USE_CUDA': '1',
'G2G': '1'
}
else:
if self.current_system.name == 'kesch':
self.valid_prog_environs = ['PrgEnv-cray-nompi',
'PrgEnv-pgi-nompi']
'PrgEnv-pgi-nompi',
'PrgEnv-gnu-nompi',
'PrgEnv-cray-c2sm',
'PrgEnv-pgi-c2sm',
'PrgEnv-gnu-c2sm']

self.num_tasks = 1
self.num_tasks_per_node = 1
Expand All @@ -61,9 +70,19 @@ def setup(self, partition, environ, **job_opts):
'-Mnorpath', '-lstdc++']
elif self.current_system.name == 'kesch':
self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0']
self.build_system.ldflags = [
'-acc', '-ta:tesla:cc35,cuda8.0', '-lstdc++',
self.build_system.ldflags = ['-acc', '-ta:tesla:cc35,cuda8.0',
'-lstdc++']
if environ.name == 'PrgEnv-pgi-nompi':
self.build_system.ldflags += [
'-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64',
'-lcublas', '-lcudart'
]
elif environ.name.startswith('PrgEnv-gnu'):
self.build_system.ldflags = ['-lstdc++']
if self.current_system.name == 'kesch':
self.build_system.ldflags += [
'-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64',
'-lcublas', '-lcudart']
'-lcublas', '-lcudart'
]

super().setup(partition, environ, **job_opts)
10 changes: 6 additions & 4 deletions cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,11 @@ program openacc_cuda_mpi_cppstd
! Call a CUDA kernel with host arrays
call call_cuda_kernel_with_copy(f1, f2, NSIZE)

#ifdef _OPENACC
! Call a CUDA kernel without data copy, use device ptr
call call_cuda_kernel_no_copy(f3, f2, NSIZE)
!$acc update host(f3)
#endif

! Call a C++ function using STD lib
call call_cpp_std(f2, NSIZE, cpp_std_sum)
Expand Down Expand Up @@ -119,8 +121,8 @@ program openacc_cuda_mpi_cppstd
subroutine call_cuda_kernel_with_copy(f1,f2,n)
use, intrinsic :: iso_c_binding
implicit none
real, intent(inout) :: f1(:)
real, intent(in) :: f2(:)
real, intent(inout), target :: f1(:)
real, intent(in), target :: f2(:)
integer, intent(in) :: n

interface
Expand All @@ -137,8 +139,8 @@ end subroutine call_cuda_kernel_with_copy
subroutine call_cuda_kernel_no_copy(f1,f2,n)
use, intrinsic :: iso_c_binding
implicit none
real, intent(inout) :: f1(:)
real, intent(in) :: f2(:)
real, intent(inout), target :: f1(:)
real, intent(in), target :: f2(:)
integer, intent(in) :: n

interface
Expand Down