diff --git a/config/cscs.py b/config/cscs.py index 38fa31644b..f04c1db133 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -131,8 +131,8 @@ class ReframeSettings: 'PrgEnv-pgi-c2sm', 'PrgEnv-gnu-c2sm', 'PrgEnv-cray-c2sm-gpu', - 'PrgEnv-gnu-c2sm-gpu', - 'PrgEnv-cray-c2sm-gpu'], + 'PrgEnv-pgi-c2sm-gpu', + 'PrgEnv-gnu-c2sm-gpu'], 'descr': 'Kesch compute nodes', 'resources': { '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], @@ -244,36 +244,42 @@ class ReframeSettings: }, 'PrgEnv-cray-c2sm': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/cray-env/base'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/cray-env/base'], }, 'PrgEnv-cray-c2sm-gpu': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/cray-env/gpu'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/cray-env/gpu'], }, 'PrgEnv-pgi-c2sm': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/pgi-env/base'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/pgi-env/base'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', }, 'PrgEnv-pgi-c2sm-gpu': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/pgi-env/gpu'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/pgi-env/gpu'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', }, 'PrgEnv-gnu-c2sm': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/gnu-env/base'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/gnu-env/base'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', }, 'PrgEnv-gnu-c2sm-gpu': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/gnu-env/gpu'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/gnu-env/gpu'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', diff --git a/cscs-checks/mch/automatic_arrays.py b/cscs-checks/mch/automatic_arrays.py index b89fd1806a..9109a522ff 100644 --- a/cscs-checks/mch/automatic_arrays.py +++ b/cscs-checks/mch/automatic_arrays.py @@ -8,11 +8,14 @@ def __init__(self): super().__init__() self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', - 'PrgEnv-gnu'] + 'PrgEnv-gnu', 'PrgEnv-cray-c2sm-gpu', + 'PrgEnv-pgi-c2sm-gpu'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] elif self.current_system.name in ['kesch']: self.modules = ['craype-accel-nvidia35'] + # FIXME: workaround -- the variable should not be needed since + # there is no GPUdirect in this check self.variables = {'MV2_USE_CUDA': '1'} # This tets requires an MPI compiler, although it uses a single task @@ -50,14 +53,20 @@ def __init__(self): self.tags = {'production'} def setup(self, partition, environ, **job_opts): - if environ.name == 'PrgEnv-cray': + if environ.name.startswith('PrgEnv-cray'): + envname = 'PrgEnv-cray' self.build_system.fflags += ['-hacc', '-hnoomp'] - elif environ.name == 'PrgEnv-pgi': + elif environ.name.startswith('PrgEnv-pgi'): + envname = 'PrgEnv-pgi' self.build_system.fflags += ['-acc'] if self.current_system.name == 'kesch': self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0'] elif self.current_system.name in ['daint', 'dom']: self.build_system.fflags += ['-ta=tesla,cc60', '-Mnorpath'] + elif environ.name.startswith('PrgEnv-gnu'): + envname = 'PrgEnv-gnu' + else: + envname = environ.name - self.reference = self.arrays_reference[environ.name] + self.reference = self.arrays_reference[envname] super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/mch/g2g_meteoswiss_check.py b/cscs-checks/mch/g2g_meteoswiss_check.py index ce2afa3c54..a0470369c6 100644 --- a/cscs-checks/mch/g2g_meteoswiss_check.py +++ b/cscs-checks/mch/g2g_meteoswiss_check.py @@ -9,6 +9,9 @@ def __init__(self, g2g): self.descr = 'G2G Meteoswiss check with G2G=%s' % g2g self.strict_check = False self.valid_systems = ['kesch:cn'] + + # FIXME: temporary workaround until the mvapich module is fixed; + # 'PrgEnv-gnu-c2sm-gpu' will be added later self.valid_prog_environs = ['PrgEnv-gnu'] self.modules = ['cmake', 'craype-accel-nvidia35'] self.pre_run = ["export EXECUTABLE=$(ls src/ | " diff --git a/cscs-checks/mch/gpu_direct_acc.py b/cscs-checks/mch/gpu_direct_acc.py index 94635541b9..ff1356f2fd 100644 --- a/cscs-checks/mch/gpu_direct_acc.py +++ b/cscs-checks/mch/gpu_direct_acc.py @@ -9,7 +9,11 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for Fortran OpenACC' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] + + # FIXME: temporary workaround until the mvapich module is fixed; + # 'PrgEnv-pgi-c2sm-gpu' will be added later + self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-cray-c2sm-gpu', + 'PrgEnv-pgi'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'} diff --git a/cscs-checks/mch/gpu_direct_cuda.py b/cscs-checks/mch/gpu_direct_cuda.py index 140709251f..3495d3c6b5 100644 --- a/cscs-checks/mch/gpu_direct_cuda.py +++ b/cscs-checks/mch/gpu_direct_cuda.py @@ -9,6 +9,8 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for CUDA' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] + # FIXME: temporary workaround until the mvapich module is fixed + # 'PrgEnv-gnu-c2sm-gpu' will be added later self.valid_prog_environs = ['PrgEnv-gnu'] self.sourcepath = 'gpu_direct_cuda.cu' self.build_system = 'SingleSource' diff --git a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py index d24efecf14..d156fc9fa5 100644 --- a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py +++ b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py @@ -10,36 +10,45 @@ def __init__(self, withmpi): self.name = 'OpenaccCudaCPP' + name_suffix self.descr = 'test for OpenACC, CUDA, MPI, and C++' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] + self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', 'PrgEnv-gnu'] self.build_system = 'Make' self.build_system.fflags = ['-O2'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] - self.variables = { - 'MPICH_RDMA_ENABLED_CUDA': '1', - 'CRAY_CUDA_MPS': '1' - } self.num_tasks = 12 self.num_tasks_per_node = 12 self.num_gpus_per_node = 1 self.build_system.options = ['NVCC_FLAGS="-arch=compute_60"'] - elif self.current_system.name in ['kesch']: + elif self.current_system.name == 'kesch': self.modules = ['craype-accel-nvidia35'] - self.variables = { - 'MV2_USE_CUDA': '1', - 'G2G': '1' - } self.num_tasks = 8 self.num_tasks_per_node = 8 self.num_gpus_per_node = 8 self.build_system.options = ['NVCC_FLAGS="-arch=compute_37"'] + # FIXME: temporary workaround until the mvapich module is fixed; + # 'PrgEnv-{pgi,gnu}-c2sm-gpu' will be added later + self.valid_prog_environs += ['PrgEnv-cray-c2sm-gpu'] if withmpi: self.build_system.cppflags = ['-DUSE_MPI'] + if self.current_system.name in ['daint', 'dom']: + self.variables = { + 'MPICH_RDMA_ENABLED_CUDA': '1', + 'CRAY_CUDA_MPS': '1' + } + elif self.current_system.name in ['kesch']: + self.variables = { + 'MV2_USE_CUDA': '1', + 'G2G': '1' + } else: if self.current_system.name == 'kesch': self.valid_prog_environs = ['PrgEnv-cray-nompi', - 'PrgEnv-pgi-nompi'] + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu-nompi', + 'PrgEnv-cray-c2sm', + 'PrgEnv-pgi-c2sm', + 'PrgEnv-gnu-c2sm'] self.num_tasks = 1 self.num_tasks_per_node = 1 @@ -61,9 +70,19 @@ def setup(self, partition, environ, **job_opts): '-Mnorpath', '-lstdc++'] elif self.current_system.name == 'kesch': self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0'] - self.build_system.ldflags = [ - '-acc', '-ta:tesla:cc35,cuda8.0', '-lstdc++', + self.build_system.ldflags = ['-acc', '-ta:tesla:cc35,cuda8.0', + '-lstdc++'] + if environ.name == 'PrgEnv-pgi-nompi': + self.build_system.ldflags += [ + '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', + '-lcublas', '-lcudart' + ] + elif environ.name.startswith('PrgEnv-gnu'): + self.build_system.ldflags = ['-lstdc++'] + if self.current_system.name == 'kesch': + self.build_system.ldflags += [ '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', - '-lcublas', '-lcudart'] + '-lcublas', '-lcudart' + ] super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 b/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 index 439f493648..44c9ffb5be 100644 --- a/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 +++ b/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 @@ -70,9 +70,11 @@ program openacc_cuda_mpi_cppstd ! Call a CUDA kernel with host arrays call call_cuda_kernel_with_copy(f1, f2, NSIZE) +#ifdef _OPENACC ! Call a CUDA kernel without data copy, use device ptr call call_cuda_kernel_no_copy(f3, f2, NSIZE) !$acc update host(f3) +#endif ! Call a C++ function using STD lib call call_cpp_std(f2, NSIZE, cpp_std_sum) @@ -119,8 +121,8 @@ program openacc_cuda_mpi_cppstd subroutine call_cuda_kernel_with_copy(f1,f2,n) use, intrinsic :: iso_c_binding implicit none - real, intent(inout) :: f1(:) - real, intent(in) :: f2(:) + real, intent(inout), target :: f1(:) + real, intent(in), target :: f2(:) integer, intent(in) :: n interface @@ -137,8 +139,8 @@ end subroutine call_cuda_kernel_with_copy subroutine call_cuda_kernel_no_copy(f1,f2,n) use, intrinsic :: iso_c_binding implicit none - real, intent(inout) :: f1(:) - real, intent(in) :: f2(:) + real, intent(inout), target :: f1(:) + real, intent(in), target :: f2(:) integer, intent(in) :: n interface