From 1ce6fb2f3a1bc1ec3420b24f56cfaa2a054e0883 Mon Sep 17 00:00:00 2001 From: ajocksch Date: Fri, 12 Oct 2018 14:26:32 +0200 Subject: [PATCH 1/6] WIP: automatic_arrays --- cscs-checks/mch/automatic_arrays.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/cscs-checks/mch/automatic_arrays.py b/cscs-checks/mch/automatic_arrays.py index b89fd1806a..ff00413009 100644 --- a/cscs-checks/mch/automatic_arrays.py +++ b/cscs-checks/mch/automatic_arrays.py @@ -8,12 +8,13 @@ def __init__(self): super().__init__() self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', - 'PrgEnv-gnu'] + 'PrgEnv-gnu', 'PrgEnv-cray-c2sm-base', + 'PrgEnv-pgi-c2sm-base', + 'PrgEnv-gnu-c2sm-base'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] elif self.current_system.name in ['kesch']: self.modules = ['craype-accel-nvidia35'] - self.variables = {'MV2_USE_CUDA': '1'} # This tets requires an MPI compiler, although it uses a single task self.num_tasks = 1 @@ -43,6 +44,21 @@ def __init__(self): 'daint:gpu': {'time': (6.4E-05, None, 0.15)}, 'dom:gpu': {'time': (6.3E-05, None, 0.15)}, 'kesch:cn': {'time': (1.4E-04, None, 0.15)}, + }, + 'PrgEnv-cray-c2sm-base': { + 'daint:gpu': {'time': (5.7E-05, None, 0.15)}, + 'dom:gpu': {'time': (5.8E-05, None, 0.15)}, + 'kesch:cn': {'time': (2.9E-04, None, 0.15)}, + }, + 'PrgEnv-gnu-c2sm-base': { + 'daint:gpu': {'time': (7.0E-03, None, 0.15)}, + 'dom:gpu': {'time': (7.3E-03, None, 0.15)}, + 'kesch:cn': {'time': (6.5E-03, None, 0.15)}, + }, + 'PrgEnv-pgi-c2sm-base': { + 'daint:gpu': {'time': (6.4E-05, None, 0.15)}, + 'dom:gpu': {'time': (6.3E-05, None, 0.15)}, + 'kesch:cn': {'time': (1.4E-04, None, 0.15)}, } } @@ -50,14 +66,18 @@ def __init__(self): self.tags = {'production'} def setup(self, partition, environ, **job_opts): - if environ.name == 'PrgEnv-cray': + if environ.name.startswith('PrgEnv-cray'): self.build_system.fflags += ['-hacc', '-hnoomp'] - elif environ.name == 'PrgEnv-pgi': + elif environ.name.startswith('PrgEnv-pgi'): self.build_system.fflags += ['-acc'] if self.current_system.name == 'kesch': self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0'] elif self.current_system.name in ['daint', 'dom']: self.build_system.fflags += ['-ta=tesla,cc60', '-Mnorpath'] + # workaround for problem with mvapich library + if environ.name == 'PrgEnv-cray': + self.variables = {'MV2_USE_CUDA': '1'} + self.reference = self.arrays_reference[environ.name] super().setup(partition, environ, **job_opts) From 706e2c7e0a996e8d2f4dec4d8b4728949c949273 Mon Sep 17 00:00:00 2001 From: ajocksch Date: Tue, 30 Oct 2018 18:10:46 +0100 Subject: [PATCH 2/6] WIP: c2sm formally introduced; segfaults --- config/cscs.py | 4 ++-- cscs-checks/mch/automatic_arrays.py | 15 +++++---------- cscs-checks/mch/g2g_meteoswiss_check.py | 2 +- cscs-checks/mch/gpu_direct_acc.py | 3 ++- cscs-checks/mch/gpu_direct_cuda.py | 2 +- cscs-checks/mch/openacc_cuda_mpi_cppstd.py | 7 ++++++- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index 2e92d54e00..5706385b92 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -131,8 +131,8 @@ class ReframeSettings: 'PrgEnv-pgi-c2sm', 'PrgEnv-gnu-c2sm', 'PrgEnv-cray-c2sm-gpu', - 'PrgEnv-gnu-c2sm-gpu', - 'PrgEnv-cray-c2sm-gpu'], + 'PrgEnv-pgi-c2sm-gpu', + 'PrgEnv-gnu-c2sm-gpu'], 'descr': 'Kesch compute nodes', 'resources': { '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}'], diff --git a/cscs-checks/mch/automatic_arrays.py b/cscs-checks/mch/automatic_arrays.py index ff00413009..cb54d37c44 100644 --- a/cscs-checks/mch/automatic_arrays.py +++ b/cscs-checks/mch/automatic_arrays.py @@ -8,9 +8,8 @@ def __init__(self): super().__init__() self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', - 'PrgEnv-gnu', 'PrgEnv-cray-c2sm-base', - 'PrgEnv-pgi-c2sm-base', - 'PrgEnv-gnu-c2sm-base'] + 'PrgEnv-gnu', 'PrgEnv-cray-c2sm-gpu', + 'PrgEnv-pgi-c2sm-gpu'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] elif self.current_system.name in ['kesch']: @@ -45,17 +44,17 @@ def __init__(self): 'dom:gpu': {'time': (6.3E-05, None, 0.15)}, 'kesch:cn': {'time': (1.4E-04, None, 0.15)}, }, - 'PrgEnv-cray-c2sm-base': { + 'PrgEnv-cray-c2sm-gpu': { 'daint:gpu': {'time': (5.7E-05, None, 0.15)}, 'dom:gpu': {'time': (5.8E-05, None, 0.15)}, 'kesch:cn': {'time': (2.9E-04, None, 0.15)}, }, - 'PrgEnv-gnu-c2sm-base': { + 'PrgEnv-gnu-c2sm-gpu': { 'daint:gpu': {'time': (7.0E-03, None, 0.15)}, 'dom:gpu': {'time': (7.3E-03, None, 0.15)}, 'kesch:cn': {'time': (6.5E-03, None, 0.15)}, }, - 'PrgEnv-pgi-c2sm-base': { + 'PrgEnv-pgi-c2sm-gpu': { 'daint:gpu': {'time': (6.4E-05, None, 0.15)}, 'dom:gpu': {'time': (6.3E-05, None, 0.15)}, 'kesch:cn': {'time': (1.4E-04, None, 0.15)}, @@ -75,9 +74,5 @@ def setup(self, partition, environ, **job_opts): elif self.current_system.name in ['daint', 'dom']: self.build_system.fflags += ['-ta=tesla,cc60', '-Mnorpath'] - # workaround for problem with mvapich library - if environ.name == 'PrgEnv-cray': - self.variables = {'MV2_USE_CUDA': '1'} - self.reference = self.arrays_reference[environ.name] super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/mch/g2g_meteoswiss_check.py b/cscs-checks/mch/g2g_meteoswiss_check.py index ce2afa3c54..a4c011b844 100644 --- a/cscs-checks/mch/g2g_meteoswiss_check.py +++ b/cscs-checks/mch/g2g_meteoswiss_check.py @@ -9,7 +9,7 @@ def __init__(self, g2g): self.descr = 'G2G Meteoswiss check with G2G=%s' % g2g self.strict_check = False self.valid_systems = ['kesch:cn'] - self.valid_prog_environs = ['PrgEnv-gnu'] + self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-gnu-c2sm-gpu'] self.modules = ['cmake', 'craype-accel-nvidia35'] self.pre_run = ["export EXECUTABLE=$(ls src/ | " "grep 'GNU.*MVAPICH.*CUDA.*kesch.*')"] diff --git a/cscs-checks/mch/gpu_direct_acc.py b/cscs-checks/mch/gpu_direct_acc.py index 94635541b9..7d0e8a2eb6 100644 --- a/cscs-checks/mch/gpu_direct_acc.py +++ b/cscs-checks/mch/gpu_direct_acc.py @@ -9,7 +9,8 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for Fortran OpenACC' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] + self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-cray-c2sm-gpu', + 'PrgEnv-pgi', 'PrgEnv-pgi-c2sm-gpu'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'} diff --git a/cscs-checks/mch/gpu_direct_cuda.py b/cscs-checks/mch/gpu_direct_cuda.py index 140709251f..ee51325e74 100644 --- a/cscs-checks/mch/gpu_direct_cuda.py +++ b/cscs-checks/mch/gpu_direct_cuda.py @@ -9,7 +9,7 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for CUDA' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - self.valid_prog_environs = ['PrgEnv-gnu'] + self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-c2sm-gpu'] self.sourcepath = 'gpu_direct_cuda.cu' self.build_system = 'SingleSource' self.build_system.ldflags = ['-lcublas', '-lcudart'] diff --git a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py index d24efecf14..85b250cbbe 100644 --- a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py +++ b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py @@ -10,7 +10,12 @@ def __init__(self, withmpi): self.name = 'OpenaccCudaCPP' + name_suffix self.descr = 'test for OpenACC, CUDA, MPI, and C++' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] + self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', + 'PrgEnv-cray-c2sm', + 'PrgEnv-cray-c2sm-gpu' + 'PrgEnv-pgi-c2sm', + 'PrgEnv-pgi-c2sm-gpu', + 'PrgEnv-gnu-c2sm'] self.build_system = 'Make' self.build_system.fflags = ['-O2'] if self.current_system.name in ['daint', 'dom']: From b5b09ff3b2f275a68346ad760b8b0a474ac7ac38 Mon Sep 17 00:00:00 2001 From: ajocksch Date: Mon, 5 Nov 2018 17:26:15 +0100 Subject: [PATCH 3/6] WIP: more green --- cscs-checks/mch/openacc_cuda_mpi_cppstd.py | 37 ++++++++++++------- .../mch/src/openacc_cuda_mpi_cppstd.F90 | 10 +++-- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py index 85b250cbbe..04ecddba0c 100644 --- a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py +++ b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py @@ -11,29 +11,20 @@ def __init__(self, withmpi): self.descr = 'test for OpenACC, CUDA, MPI, and C++' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', - 'PrgEnv-cray-c2sm', - 'PrgEnv-cray-c2sm-gpu' - 'PrgEnv-pgi-c2sm', + 'PrgEnv-gnu', + 'PrgEnv-cray-c2sm-gpu', 'PrgEnv-pgi-c2sm-gpu', - 'PrgEnv-gnu-c2sm'] + 'PrgEnv-gnu-c2sm-gpu'] self.build_system = 'Make' self.build_system.fflags = ['-O2'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] - self.variables = { - 'MPICH_RDMA_ENABLED_CUDA': '1', - 'CRAY_CUDA_MPS': '1' - } self.num_tasks = 12 self.num_tasks_per_node = 12 self.num_gpus_per_node = 1 self.build_system.options = ['NVCC_FLAGS="-arch=compute_60"'] elif self.current_system.name in ['kesch']: self.modules = ['craype-accel-nvidia35'] - self.variables = { - 'MV2_USE_CUDA': '1', - 'G2G': '1' - } self.num_tasks = 8 self.num_tasks_per_node = 8 self.num_gpus_per_node = 8 @@ -41,10 +32,24 @@ def __init__(self, withmpi): if withmpi: self.build_system.cppflags = ['-DUSE_MPI'] + if self.current_system.name in ['daint', 'dom']: + self.variables = { + 'MPICH_RDMA_ENABLED_CUDA': '1', + 'CRAY_CUDA_MPS': '1' + } + elif self.current_system.name in ['kesch']: + self.variables = { + 'MV2_USE_CUDA': '1', + 'G2G': '1' + } else: if self.current_system.name == 'kesch': self.valid_prog_environs = ['PrgEnv-cray-nompi', - 'PrgEnv-pgi-nompi'] + 'PrgEnv-pgi-nompi', + 'PrgEnv-gnu-nompi', + 'PrgEnv-cray-c2sm', + 'PrgEnv-pgi-c2sm', + 'PrgEnv-gnu-c2sm'] self.num_tasks = 1 self.num_tasks_per_node = 1 @@ -70,5 +75,11 @@ def setup(self, partition, environ, **job_opts): '-acc', '-ta:tesla:cc35,cuda8.0', '-lstdc++', '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', '-lcublas', '-lcudart'] + elif environ.name.startswith('PrgEnv-gnu'): + self.build_system.ldflags = ['-lstdc++'] + if self.current_system.name == 'kesch': + self.build_system.ldflags += [ + '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', + '-lcublas', '-lcudart'] super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 b/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 index 439f493648..44c9ffb5be 100644 --- a/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 +++ b/cscs-checks/mch/src/openacc_cuda_mpi_cppstd.F90 @@ -70,9 +70,11 @@ program openacc_cuda_mpi_cppstd ! Call a CUDA kernel with host arrays call call_cuda_kernel_with_copy(f1, f2, NSIZE) +#ifdef _OPENACC ! Call a CUDA kernel without data copy, use device ptr call call_cuda_kernel_no_copy(f3, f2, NSIZE) !$acc update host(f3) +#endif ! Call a C++ function using STD lib call call_cpp_std(f2, NSIZE, cpp_std_sum) @@ -119,8 +121,8 @@ program openacc_cuda_mpi_cppstd subroutine call_cuda_kernel_with_copy(f1,f2,n) use, intrinsic :: iso_c_binding implicit none - real, intent(inout) :: f1(:) - real, intent(in) :: f2(:) + real, intent(inout), target :: f1(:) + real, intent(in), target :: f2(:) integer, intent(in) :: n interface @@ -137,8 +139,8 @@ end subroutine call_cuda_kernel_with_copy subroutine call_cuda_kernel_no_copy(f1,f2,n) use, intrinsic :: iso_c_binding implicit none - real, intent(inout) :: f1(:) - real, intent(in) :: f2(:) + real, intent(inout), target :: f1(:) + real, intent(in), target :: f2(:) integer, intent(in) :: n interface From f3c9c654735d3f43f3a9c34a7bf19b507d5a11dd Mon Sep 17 00:00:00 2001 From: ajocksch Date: Tue, 6 Nov 2018 15:13:04 +0100 Subject: [PATCH 4/6] all checks green --- cscs-checks/mch/automatic_arrays.py | 2 ++ cscs-checks/mch/g2g_meteoswiss_check.py | 5 ++++- cscs-checks/mch/gpu_direct_acc.py | 5 ++++- cscs-checks/mch/gpu_direct_cuda.py | 5 ++++- cscs-checks/mch/openacc_cuda_mpi_cppstd.py | 6 ++++-- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cscs-checks/mch/automatic_arrays.py b/cscs-checks/mch/automatic_arrays.py index cb54d37c44..c7aa63cdae 100644 --- a/cscs-checks/mch/automatic_arrays.py +++ b/cscs-checks/mch/automatic_arrays.py @@ -14,6 +14,8 @@ def __init__(self): self.modules = ['craype-accel-nvidia60'] elif self.current_system.name in ['kesch']: self.modules = ['craype-accel-nvidia35'] + # workaround + self.variables = {'MV2_USE_CUDA': '1'} # This tets requires an MPI compiler, although it uses a single task self.num_tasks = 1 diff --git a/cscs-checks/mch/g2g_meteoswiss_check.py b/cscs-checks/mch/g2g_meteoswiss_check.py index a4c011b844..0d1f9e80ff 100644 --- a/cscs-checks/mch/g2g_meteoswiss_check.py +++ b/cscs-checks/mch/g2g_meteoswiss_check.py @@ -9,7 +9,10 @@ def __init__(self, g2g): self.descr = 'G2G Meteoswiss check with G2G=%s' % g2g self.strict_check = False self.valid_systems = ['kesch:cn'] - self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-gnu-c2sm-gpu'] + self.valid_prog_environs = ['PrgEnv-gnu', +# temporary workaround until the mvapich module is fixed +# 'PrgEnv-gnu-c2sm-gpu', + ] self.modules = ['cmake', 'craype-accel-nvidia35'] self.pre_run = ["export EXECUTABLE=$(ls src/ | " "grep 'GNU.*MVAPICH.*CUDA.*kesch.*')"] diff --git a/cscs-checks/mch/gpu_direct_acc.py b/cscs-checks/mch/gpu_direct_acc.py index 7d0e8a2eb6..7826c2bde3 100644 --- a/cscs-checks/mch/gpu_direct_acc.py +++ b/cscs-checks/mch/gpu_direct_acc.py @@ -10,7 +10,10 @@ def __init__(self): self.descr = 'tests gpu-direct for Fortran OpenACC' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-cray-c2sm-gpu', - 'PrgEnv-pgi', 'PrgEnv-pgi-c2sm-gpu'] + 'PrgEnv-pgi', +# temporary workaround until the mvapich module is fixed +# 'PrgEnv-pgi-c2sm-gpu', + ] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'} diff --git a/cscs-checks/mch/gpu_direct_cuda.py b/cscs-checks/mch/gpu_direct_cuda.py index ee51325e74..3c0d3a2dea 100644 --- a/cscs-checks/mch/gpu_direct_cuda.py +++ b/cscs-checks/mch/gpu_direct_cuda.py @@ -9,7 +9,10 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for CUDA' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-c2sm-gpu'] + self.valid_prog_environs = ['PrgEnv-gnu', +# temporary workaround until the mvapich module is fixed +# 'PrgEnv-c2sm-gpu', + ] self.sourcepath = 'gpu_direct_cuda.cu' self.build_system = 'SingleSource' self.build_system.ldflags = ['-lcublas', '-lcudart'] diff --git a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py index 04ecddba0c..708c50ca28 100644 --- a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py +++ b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py @@ -13,8 +13,10 @@ def __init__(self, withmpi): self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', 'PrgEnv-gnu', 'PrgEnv-cray-c2sm-gpu', - 'PrgEnv-pgi-c2sm-gpu', - 'PrgEnv-gnu-c2sm-gpu'] +# temporary workaround until the mvapich module is fixed +# 'PrgEnv-pgi-c2sm-gpu', +# 'PrgEnv-gnu-c2sm-gpu', + ] self.build_system = 'Make' self.build_system.fflags = ['-O2'] if self.current_system.name in ['daint', 'dom']: From adb04e580ef7339e6106502c7d08d1d76504a2d6 Mon Sep 17 00:00:00 2001 From: ajocksch Date: Wed, 21 Nov 2018 11:17:41 +0100 Subject: [PATCH 5/6] incorporated changes requested --- cscs-checks/mch/automatic_arrays.py | 29 +++++++++------------- cscs-checks/mch/g2g_meteoswiss_check.py | 8 +++--- cscs-checks/mch/gpu_direct_acc.py | 8 +++--- cscs-checks/mch/gpu_direct_cuda.py | 8 +++--- cscs-checks/mch/openacc_cuda_mpi_cppstd.py | 17 +++++++------ 5 files changed, 34 insertions(+), 36 deletions(-) diff --git a/cscs-checks/mch/automatic_arrays.py b/cscs-checks/mch/automatic_arrays.py index c7aa63cdae..bbdb380657 100644 --- a/cscs-checks/mch/automatic_arrays.py +++ b/cscs-checks/mch/automatic_arrays.py @@ -14,7 +14,8 @@ def __init__(self): self.modules = ['craype-accel-nvidia60'] elif self.current_system.name in ['kesch']: self.modules = ['craype-accel-nvidia35'] - # workaround + # FIXME: workaround -- the variable should not be needed since + # there is no gpu-direct in the check self.variables = {'MV2_USE_CUDA': '1'} # This tets requires an MPI compiler, although it uses a single task @@ -46,21 +47,6 @@ def __init__(self): 'dom:gpu': {'time': (6.3E-05, None, 0.15)}, 'kesch:cn': {'time': (1.4E-04, None, 0.15)}, }, - 'PrgEnv-cray-c2sm-gpu': { - 'daint:gpu': {'time': (5.7E-05, None, 0.15)}, - 'dom:gpu': {'time': (5.8E-05, None, 0.15)}, - 'kesch:cn': {'time': (2.9E-04, None, 0.15)}, - }, - 'PrgEnv-gnu-c2sm-gpu': { - 'daint:gpu': {'time': (7.0E-03, None, 0.15)}, - 'dom:gpu': {'time': (7.3E-03, None, 0.15)}, - 'kesch:cn': {'time': (6.5E-03, None, 0.15)}, - }, - 'PrgEnv-pgi-c2sm-gpu': { - 'daint:gpu': {'time': (6.4E-05, None, 0.15)}, - 'dom:gpu': {'time': (6.3E-05, None, 0.15)}, - 'kesch:cn': {'time': (1.4E-04, None, 0.15)}, - } } self.maintainers = ['AJ', 'VK'] @@ -76,5 +62,14 @@ def setup(self, partition, environ, **job_opts): elif self.current_system.name in ['daint', 'dom']: self.build_system.fflags += ['-ta=tesla,cc60', '-Mnorpath'] - self.reference = self.arrays_reference[environ.name] + if environ.name.startswith('PrgEnv-cray'): + envname = 'PrgEnv-cray' + elif environ.name.startswith('PrgEnv-gnu'): + envname = 'PrgEnv-gnu' + elif environ.name.startswith('PrgEnv-pgi'): + envname = 'PrgEnv-pgi' + else: + envname = environ.name + + self.reference = self.arrays_reference[envname] super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/mch/g2g_meteoswiss_check.py b/cscs-checks/mch/g2g_meteoswiss_check.py index 0d1f9e80ff..796ef1b3a2 100644 --- a/cscs-checks/mch/g2g_meteoswiss_check.py +++ b/cscs-checks/mch/g2g_meteoswiss_check.py @@ -9,10 +9,10 @@ def __init__(self, g2g): self.descr = 'G2G Meteoswiss check with G2G=%s' % g2g self.strict_check = False self.valid_systems = ['kesch:cn'] - self.valid_prog_environs = ['PrgEnv-gnu', -# temporary workaround until the mvapich module is fixed -# 'PrgEnv-gnu-c2sm-gpu', - ] + # FIXME: temporary workaround until the mvapich module is fixed + # afterwards 'PrgEnv-gnu-c2sm-gpu' will be added which is not + # implemented now + self.valid_prog_environs = ['PrgEnv-gnu'] self.modules = ['cmake', 'craype-accel-nvidia35'] self.pre_run = ["export EXECUTABLE=$(ls src/ | " "grep 'GNU.*MVAPICH.*CUDA.*kesch.*')"] diff --git a/cscs-checks/mch/gpu_direct_acc.py b/cscs-checks/mch/gpu_direct_acc.py index 7826c2bde3..dde0ec0f00 100644 --- a/cscs-checks/mch/gpu_direct_acc.py +++ b/cscs-checks/mch/gpu_direct_acc.py @@ -9,11 +9,11 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for Fortran OpenACC' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] + # FIXME: temporary workaround until the mvapich module is fixed + # afterwards 'PrgEnv-pgi-c2sm-gpu' will be added which is not + # implemented now self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-cray-c2sm-gpu', - 'PrgEnv-pgi', -# temporary workaround until the mvapich module is fixed -# 'PrgEnv-pgi-c2sm-gpu', - ] + 'PrgEnv-pgi'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'} diff --git a/cscs-checks/mch/gpu_direct_cuda.py b/cscs-checks/mch/gpu_direct_cuda.py index 3c0d3a2dea..e1af93c290 100644 --- a/cscs-checks/mch/gpu_direct_cuda.py +++ b/cscs-checks/mch/gpu_direct_cuda.py @@ -9,10 +9,10 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for CUDA' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - self.valid_prog_environs = ['PrgEnv-gnu', -# temporary workaround until the mvapich module is fixed -# 'PrgEnv-c2sm-gpu', - ] + # FIXME: temporary workaround until the mvapich module is fixed + # afterwards 'PrgEnv-gnu-c2sm-gpu' will be added which is not + # implemented now + self.valid_prog_environs = ['PrgEnv-gnu'] self.sourcepath = 'gpu_direct_cuda.cu' self.build_system = 'SingleSource' self.build_system.ldflags = ['-lcublas', '-lcudart'] diff --git a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py index 708c50ca28..70130cdaea 100644 --- a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py +++ b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py @@ -10,12 +10,12 @@ def __init__(self, withmpi): self.name = 'OpenaccCudaCPP' + name_suffix self.descr = 'test for OpenACC, CUDA, MPI, and C++' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] + # FIXME: temporary workaround until the mvapich module is fixed + # afterwards 'PrgEnv-pgi-c2sm-gpu' and 'PrgEnv-gnu-c2sm-gpu' + # will be added which is not implemented now self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', 'PrgEnv-gnu', 'PrgEnv-cray-c2sm-gpu', -# temporary workaround until the mvapich module is fixed -# 'PrgEnv-pgi-c2sm-gpu', -# 'PrgEnv-gnu-c2sm-gpu', ] self.build_system = 'Make' self.build_system.fflags = ['-O2'] @@ -74,12 +74,15 @@ def setup(self, partition, environ, **job_opts): elif self.current_system.name == 'kesch': self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0'] self.build_system.ldflags = [ - '-acc', '-ta:tesla:cc35,cuda8.0', '-lstdc++', - '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', - '-lcublas', '-lcudart'] + '-acc', '-ta:tesla:cc35,cuda8.0', '-lstdc++'] + if environ.name == 'PrgEnv-pgi-nompi': + self.build_system.ldflags += [ + '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', + '-lcublas', '-lcudart'] elif environ.name.startswith('PrgEnv-gnu'): self.build_system.ldflags = ['-lstdc++'] - if self.current_system.name == 'kesch': + if self.current_system.name == 'kesch' and (environ.name == + 'PrgEnv-gnu-nompi' or environ.name == 'PrgEnv-gnu-c2sm'): self.build_system.ldflags += [ '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', '-lcublas', '-lcudart'] From ba6d0b14f388b895b5dd224c1cc2dd1f5022a93a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 23 Nov 2018 09:52:15 +0100 Subject: [PATCH 6/6] Fine tune MCH tests and redefine C2SM environments --- config/cscs.py | 18 ++++++++++----- cscs-checks/mch/automatic_arrays.py | 11 ++++----- cscs-checks/mch/g2g_meteoswiss_check.py | 6 ++--- cscs-checks/mch/gpu_direct_acc.py | 6 ++--- cscs-checks/mch/gpu_direct_cuda.py | 3 +-- cscs-checks/mch/openacc_cuda_mpi_cppstd.py | 26 ++++++++++------------ 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/config/cscs.py b/config/cscs.py index 5706385b92..4725d1a8e1 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -244,36 +244,42 @@ class ReframeSettings: }, 'PrgEnv-cray-c2sm': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/cray-env/base'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/cray-env/base'], }, 'PrgEnv-cray-c2sm-gpu': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/cray-env/gpu'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/cray-env/gpu'], }, 'PrgEnv-pgi-c2sm': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/pgi-env/base'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/pgi-env/base'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', }, 'PrgEnv-pgi-c2sm-gpu': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/pgi-env/gpu'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/pgi-env/gpu'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', }, 'PrgEnv-gnu-c2sm': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/gnu-env/base'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/gnu-env/base'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', }, 'PrgEnv-gnu-c2sm-gpu': { 'type': 'ProgEnvironment', - 'modules': ['env', 'c2sm/gnu-env/gpu'], + 'modules': ['c2sm-rcm/1.00.00-kesch', + 'c2sm/gnu-env/gpu'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', diff --git a/cscs-checks/mch/automatic_arrays.py b/cscs-checks/mch/automatic_arrays.py index bbdb380657..9109a522ff 100644 --- a/cscs-checks/mch/automatic_arrays.py +++ b/cscs-checks/mch/automatic_arrays.py @@ -15,7 +15,7 @@ def __init__(self): elif self.current_system.name in ['kesch']: self.modules = ['craype-accel-nvidia35'] # FIXME: workaround -- the variable should not be needed since - # there is no gpu-direct in the check + # there is no GPUdirect in this check self.variables = {'MV2_USE_CUDA': '1'} # This tets requires an MPI compiler, although it uses a single task @@ -46,7 +46,7 @@ def __init__(self): 'daint:gpu': {'time': (6.4E-05, None, 0.15)}, 'dom:gpu': {'time': (6.3E-05, None, 0.15)}, 'kesch:cn': {'time': (1.4E-04, None, 0.15)}, - }, + } } self.maintainers = ['AJ', 'VK'] @@ -54,20 +54,17 @@ def __init__(self): def setup(self, partition, environ, **job_opts): if environ.name.startswith('PrgEnv-cray'): + envname = 'PrgEnv-cray' self.build_system.fflags += ['-hacc', '-hnoomp'] elif environ.name.startswith('PrgEnv-pgi'): + envname = 'PrgEnv-pgi' self.build_system.fflags += ['-acc'] if self.current_system.name == 'kesch': self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0'] elif self.current_system.name in ['daint', 'dom']: self.build_system.fflags += ['-ta=tesla,cc60', '-Mnorpath'] - - if environ.name.startswith('PrgEnv-cray'): - envname = 'PrgEnv-cray' elif environ.name.startswith('PrgEnv-gnu'): envname = 'PrgEnv-gnu' - elif environ.name.startswith('PrgEnv-pgi'): - envname = 'PrgEnv-pgi' else: envname = environ.name diff --git a/cscs-checks/mch/g2g_meteoswiss_check.py b/cscs-checks/mch/g2g_meteoswiss_check.py index 796ef1b3a2..a0470369c6 100644 --- a/cscs-checks/mch/g2g_meteoswiss_check.py +++ b/cscs-checks/mch/g2g_meteoswiss_check.py @@ -9,9 +9,9 @@ def __init__(self, g2g): self.descr = 'G2G Meteoswiss check with G2G=%s' % g2g self.strict_check = False self.valid_systems = ['kesch:cn'] - # FIXME: temporary workaround until the mvapich module is fixed - # afterwards 'PrgEnv-gnu-c2sm-gpu' will be added which is not - # implemented now + + # FIXME: temporary workaround until the mvapich module is fixed; + # 'PrgEnv-gnu-c2sm-gpu' will be added later self.valid_prog_environs = ['PrgEnv-gnu'] self.modules = ['cmake', 'craype-accel-nvidia35'] self.pre_run = ["export EXECUTABLE=$(ls src/ | " diff --git a/cscs-checks/mch/gpu_direct_acc.py b/cscs-checks/mch/gpu_direct_acc.py index dde0ec0f00..ff1356f2fd 100644 --- a/cscs-checks/mch/gpu_direct_acc.py +++ b/cscs-checks/mch/gpu_direct_acc.py @@ -9,9 +9,9 @@ def __init__(self): super().__init__() self.descr = 'tests gpu-direct for Fortran OpenACC' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - # FIXME: temporary workaround until the mvapich module is fixed - # afterwards 'PrgEnv-pgi-c2sm-gpu' will be added which is not - # implemented now + + # FIXME: temporary workaround until the mvapich module is fixed; + # 'PrgEnv-pgi-c2sm-gpu' will be added later self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-cray-c2sm-gpu', 'PrgEnv-pgi'] if self.current_system.name in ['daint', 'dom']: diff --git a/cscs-checks/mch/gpu_direct_cuda.py b/cscs-checks/mch/gpu_direct_cuda.py index e1af93c290..3495d3c6b5 100644 --- a/cscs-checks/mch/gpu_direct_cuda.py +++ b/cscs-checks/mch/gpu_direct_cuda.py @@ -10,8 +10,7 @@ def __init__(self): self.descr = 'tests gpu-direct for CUDA' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] # FIXME: temporary workaround until the mvapich module is fixed - # afterwards 'PrgEnv-gnu-c2sm-gpu' will be added which is not - # implemented now + # 'PrgEnv-gnu-c2sm-gpu' will be added later self.valid_prog_environs = ['PrgEnv-gnu'] self.sourcepath = 'gpu_direct_cuda.cu' self.build_system = 'SingleSource' diff --git a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py index 70130cdaea..d156fc9fa5 100644 --- a/cscs-checks/mch/openacc_cuda_mpi_cppstd.py +++ b/cscs-checks/mch/openacc_cuda_mpi_cppstd.py @@ -10,13 +10,7 @@ def __init__(self, withmpi): self.name = 'OpenaccCudaCPP' + name_suffix self.descr = 'test for OpenACC, CUDA, MPI, and C++' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] - # FIXME: temporary workaround until the mvapich module is fixed - # afterwards 'PrgEnv-pgi-c2sm-gpu' and 'PrgEnv-gnu-c2sm-gpu' - # will be added which is not implemented now - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', - 'PrgEnv-gnu', - 'PrgEnv-cray-c2sm-gpu', - ] + self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi', 'PrgEnv-gnu'] self.build_system = 'Make' self.build_system.fflags = ['-O2'] if self.current_system.name in ['daint', 'dom']: @@ -25,12 +19,15 @@ def __init__(self, withmpi): self.num_tasks_per_node = 12 self.num_gpus_per_node = 1 self.build_system.options = ['NVCC_FLAGS="-arch=compute_60"'] - elif self.current_system.name in ['kesch']: + elif self.current_system.name == 'kesch': self.modules = ['craype-accel-nvidia35'] self.num_tasks = 8 self.num_tasks_per_node = 8 self.num_gpus_per_node = 8 self.build_system.options = ['NVCC_FLAGS="-arch=compute_37"'] + # FIXME: temporary workaround until the mvapich module is fixed; + # 'PrgEnv-{pgi,gnu}-c2sm-gpu' will be added later + self.valid_prog_environs += ['PrgEnv-cray-c2sm-gpu'] if withmpi: self.build_system.cppflags = ['-DUSE_MPI'] @@ -73,18 +70,19 @@ def setup(self, partition, environ, **job_opts): '-Mnorpath', '-lstdc++'] elif self.current_system.name == 'kesch': self.build_system.fflags += ['-ta=tesla,cc35,cuda8.0'] - self.build_system.ldflags = [ - '-acc', '-ta:tesla:cc35,cuda8.0', '-lstdc++'] + self.build_system.ldflags = ['-acc', '-ta:tesla:cc35,cuda8.0', + '-lstdc++'] if environ.name == 'PrgEnv-pgi-nompi': self.build_system.ldflags += [ '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', - '-lcublas', '-lcudart'] + '-lcublas', '-lcudart' + ] elif environ.name.startswith('PrgEnv-gnu'): self.build_system.ldflags = ['-lstdc++'] - if self.current_system.name == 'kesch' and (environ.name == - 'PrgEnv-gnu-nompi' or environ.name == 'PrgEnv-gnu-c2sm'): + if self.current_system.name == 'kesch': self.build_system.ldflags += [ '-L/global/opt/nvidia/cudatoolkit/8.0.61/lib64', - '-lcublas', '-lcudart'] + '-lcublas', '-lcudart' + ] super().setup(partition, environ, **job_opts)