From 11b46c8ff7f54b9186966c932cbe85ec95692ee2 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Sat, 15 Sep 2018 11:50:53 +0200 Subject: [PATCH 1/3] Adapt gpu direct test to build systems --- cscs-checks/prgenv/gpu_direct_acc.py | 13 ++++++----- cscs-checks/prgenv/gpu_direct_cuda.py | 32 ++++++++++----------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/cscs-checks/prgenv/gpu_direct_acc.py b/cscs-checks/prgenv/gpu_direct_acc.py index 56e7d4cf47..1c234f0704 100644 --- a/cscs-checks/prgenv/gpu_direct_acc.py +++ b/cscs-checks/prgenv/gpu_direct_acc.py @@ -4,21 +4,19 @@ @rfm.simple_test class GpuDirectAccCheck(rfm.RegressionTest): - def __init__(self, **kwargs): + def __init__(self): super().__init__() self.descr = 'tests gpu-direct for Fortran OpenACC' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] self.valid_prog_environs = ['PrgEnv-cray*', 'PrgEnv-pgi*'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] - self._pgi_flags = '-acc -ta=tesla:cc60 -Mnorpath' self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'} self.num_tasks = 2 self.num_gpus_per_node = 1 self.num_tasks_per_node = 1 elif self.current_system.name in ['kesch']: self.modules = ['craype-accel-nvidia35'] - self._pgi_flags = '-acc -ta=tesla:cc35' self.variables = { 'MPICH_RDMA_ENABLED_CUDA': '1', 'MV2_USE_CUDA': '1', @@ -29,6 +27,7 @@ def __init__(self, **kwargs): self.num_tasks_per_node = 8 self.sourcepath = 'gpu_direct_acc.F90' + self.build_system = 'SingleSource' self.sanity_patterns = sn.all([ sn.assert_found(r'GPU with OpenACC', self.stdout), sn.assert_found(r'Result :\s+OK', self.stdout) @@ -39,8 +38,12 @@ def __init__(self, **kwargs): def setup(self, partition, environ, **job_opts): if environ.name.startswith('PrgEnv-cray'): - environ.fflags = '-hacc -hnoomp' + self.build_system.fflags = ['-hacc', '-hnoomp'] elif environ.name.startswith('PrgEnv-pgi'): - environ.fflags = self._pgi_flags + if self.current_system.name in ['daint', 'dom']: + self.build_system.fflags = ['-acc', '-ta=tesla:cc60', + '-Mnorpath'] + elif self.current_system.name == 'kesch': + self.build_system.fflags = ['-acc', '-ta=tesla:cc35'] super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/prgenv/gpu_direct_cuda.py b/cscs-checks/prgenv/gpu_direct_cuda.py index 771be65846..4c300dde2e 100644 --- a/cscs-checks/prgenv/gpu_direct_cuda.py +++ b/cscs-checks/prgenv/gpu_direct_cuda.py @@ -4,47 +4,39 @@ @rfm.simple_test class GpuDirectCudaCheck(rfm.RegressionTest): - def __init__(self, **kwargs): + def __init__(self): super().__init__() self.descr = 'tests gpu-direct for CUDA' self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn'] self.valid_prog_environs = ['PrgEnv-gnu'] + self.sourcepath = 'gpu_direct_cuda.cu' + self.build_system = 'SingleSource' if self.current_system.name in ['daint', 'dom']: + self.modules = ['craype-accel-nvidia60'] self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'} - elif self.current_system.name in ['kesch']: + self.build_system.cxxflags = ['-ccbin CC', '-lcublas', '-lcudart', + '-arch=sm_60'] + elif self.current_system.name == 'kesch': + self.modules = ['cudatoolkit'] self.valid_prog_environs = ['PrgEnv-gnu-gdr'] self.variables = { 'MPICH_RDMA_ENABLED_CUDA': '1', 'MV2_USE_CUDA': '1', 'G2G': '1', } + self.build_system.cxxflags = ['-ccbin mpicxx', '-lcublas', + '-lcudart', '-arch=sm_60'] self.num_tasks = 2 self.num_gpus_per_node = 1 - self.sourcepath = 'gpu_direct_cuda.cu' self.num_tasks_per_node = 1 - self.modules = ['cudatoolkit'] - result = sn.extractsingle(r'Result :\s+(?P\d+\.?\d*)', self.stdout, 'result', float) self.sanity_patterns = sn.assert_reference(result, 1., -1e-5, 1e-5) - self.maintainers = ['AJ', 'VK'] - self.tags = {'production'} self.pre_run = [ 'export LD_PRELOAD=' '$(pkg-config --variable=libdir mvapich2-gdr)/libmpi.so' ] - def compile(self): - # Set nvcc flags - if self.current_system.name == 'kesch': - nvidia_sm = '37' - cpp_compiler = 'mpicxx' - else: - nvidia_sm = '60' - cpp_compiler = 'CC' - - self.current_environ.cxxflags = ( - '-ccbin %s -lcublas -lcudart -arch=sm_%s' % - (cpp_compiler, nvidia_sm)) - super().compile() + self.maintainers = ['AJ', 'VK'] + self.tags = {'production'} From 093b13008093a64fc68f09b77b0cb9b0c9da5eee Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 18 Sep 2018 08:05:35 +0200 Subject: [PATCH 2/3] Address PR comments --- cscs-checks/prgenv/gpu_direct_acc.py | 6 +++--- cscs-checks/prgenv/gpu_direct_cuda.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cscs-checks/prgenv/gpu_direct_acc.py b/cscs-checks/prgenv/gpu_direct_acc.py index 1c234f0704..765cb664e8 100644 --- a/cscs-checks/prgenv/gpu_direct_acc.py +++ b/cscs-checks/prgenv/gpu_direct_acc.py @@ -40,10 +40,10 @@ def setup(self, partition, environ, **job_opts): if environ.name.startswith('PrgEnv-cray'): self.build_system.fflags = ['-hacc', '-hnoomp'] elif environ.name.startswith('PrgEnv-pgi'): + self.build_system.fflags = ['-acc'] if self.current_system.name in ['daint', 'dom']: - self.build_system.fflags = ['-acc', '-ta=tesla:cc60', - '-Mnorpath'] + self.build_system.fflags += ['-ta=tesla:cc60', '-Mnorpath'] elif self.current_system.name == 'kesch': - self.build_system.fflags = ['-acc', '-ta=tesla:cc35'] + self.build_system.fflags += ['-ta=tesla:cc35'] super().setup(partition, environ, **job_opts) diff --git a/cscs-checks/prgenv/gpu_direct_cuda.py b/cscs-checks/prgenv/gpu_direct_cuda.py index 4c300dde2e..2797ef9ae9 100644 --- a/cscs-checks/prgenv/gpu_direct_cuda.py +++ b/cscs-checks/prgenv/gpu_direct_cuda.py @@ -11,11 +11,11 @@ def __init__(self): self.valid_prog_environs = ['PrgEnv-gnu'] self.sourcepath = 'gpu_direct_cuda.cu' self.build_system = 'SingleSource' + self.build_system.ldflags = ['-lcublas', '-lcudart'] if self.current_system.name in ['daint', 'dom']: self.modules = ['craype-accel-nvidia60'] self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'} - self.build_system.cxxflags = ['-ccbin CC', '-lcublas', '-lcudart', - '-arch=sm_60'] + self.build_system.cxxflags = ['-ccbin CC', '-arch=sm_60'] elif self.current_system.name == 'kesch': self.modules = ['cudatoolkit'] self.valid_prog_environs = ['PrgEnv-gnu-gdr'] @@ -24,8 +24,7 @@ def __init__(self): 'MV2_USE_CUDA': '1', 'G2G': '1', } - self.build_system.cxxflags = ['-ccbin mpicxx', '-lcublas', - '-lcudart', '-arch=sm_60'] + self.build_system.cxxflags = ['-ccbin mpicxx', '-arch=sm_47'] self.num_tasks = 2 self.num_gpus_per_node = 1 From c04dcc10f99fb69182ff3141de09ea62094347ce Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 18 Sep 2018 13:30:06 +0200 Subject: [PATCH 3/3] Fix bug for sm_37 --- cscs-checks/prgenv/gpu_direct_cuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cscs-checks/prgenv/gpu_direct_cuda.py b/cscs-checks/prgenv/gpu_direct_cuda.py index 2797ef9ae9..0c5788ef6a 100644 --- a/cscs-checks/prgenv/gpu_direct_cuda.py +++ b/cscs-checks/prgenv/gpu_direct_cuda.py @@ -24,7 +24,7 @@ def __init__(self): 'MV2_USE_CUDA': '1', 'G2G': '1', } - self.build_system.cxxflags = ['-ccbin mpicxx', '-arch=sm_47'] + self.build_system.cxxflags = ['-ccbin mpicxx', '-arch=sm_37'] self.num_tasks = 2 self.num_gpus_per_node = 1