Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

update CP2K easyblock for compatibility with CP2K v2023.1 release (GCC only) + fix GPU support #2918

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

update CP2K easyblock for compatibility with CP2K v2023.1 release (GCC only) + fix GPU support #2918

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ef95a4c
CP2K 2023 release compatibility (gcc only). Intended for new GPU feat…
osbama Apr 14, 2023
5929a44
pep check
osbama Apr 16, 2023
8750195
pep check-line length
osbama Apr 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
158 changes: 106 additions & 52 deletions easybuild/easyblocks/c/cp2k.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
@author: Damian Alvarez (Forschungszentrum Juelich GmbH)
@author: Alan O'Cais (Forschungszentrum Juelich GmbH)
@author: Balazs Hajgato (Free University Brussels (VUB))
@author: O. Baris Malcioglu (Middle East Technical University)
"""

import fileinput
Expand Down Expand Up @@ -80,6 +81,7 @@ def __init__(self, *args, **kwargs):

# used for both libsmm and libxsmm
self.libsmm = ''
self.libsmm_path = ''
self.modincpath = ''
self.openmp = ''

Expand All @@ -103,14 +105,16 @@ def extra_options():
'plumed': [None, "Enable PLUMED support", CUSTOM],
'type': ['popt', "Type of build ('popt' or 'psmp')", CUSTOM],
'typeopt': [True, "Enable optimization", CUSTOM],
'v2023': [False, "2023 version of CP2K", CUSTOM],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should not be a config option, use LooseVersion(get_software_version('CP2K')) to figure out if it is 2023 or newer instead.

'gpuver': ['A100', "CUDA gpu version", CUSTOM],
}
return EasyBlock.extra_options(extra_vars)

def _generate_makefile(self, options):
"""Generate Makefile based on options dictionary and optional make instructions"""

text = "# Makefile generated by CP2K easyblock in EasyBuild\n"
for key, value in sorted(options.items()):
for key, value in options.items():
text += "%s = %s\n" % (key, value)
return text + self.make_instructions

Expand All @@ -124,6 +128,10 @@ def configure_step(self):
if self.cfg['type'] not in known_types:
raise EasyBuildError("Unknown build type specified: '%s', known types are %s",
self.cfg['type'], known_types)
if self.cfg['v2023'] and self.cfg['type'] == 'popt':
self.cfg['type'] = 'psmp'
setvar('OMP_NUM_THREADS', '1')
self.log.debug('In 2023 version popt is psmp with OMP_NUM_THREADS set to 1')

# correct start dir, if needed
# recent CP2K versions have a 'cp2k' dir in the unpacked 'cp2k' dir
Expand Down Expand Up @@ -154,13 +162,15 @@ def configure_step(self):
if libxsmm:
self.cfg.update('extradflags', '-D__LIBXSMM')
self.libsmm = '-lxsmm -lxsmmf'
self.libsmm_path = libxsmm
self.log.debug('Using libxsmm %s' % libxsmm)
elif libsmm:
libsmms = glob.glob(os.path.join(libsmm, 'lib', 'libsmm_*nn.a'))
dfs = [os.path.basename(os.path.splitext(x)[0]).replace('lib', '-D__HAS_') for x in libsmms]
moredflags = ' ' + ' '.join(dfs)
self.cfg.update('extradflags', moredflags)
self.libsmm = ' '.join(libsmms)
self.libsmm_path = libsmm
self.log.debug('Using libsmm %s (extradflags %s)' % (self.libsmm, moredflags))

# obtain list of modinc's to use
Expand Down Expand Up @@ -219,22 +229,39 @@ def configure_step(self):
if elpa:
options['LIBS'] += ' -lelpa'
elpa_inc_dir = os.path.join(elpa, 'include', 'elpa-%s' % get_software_version('ELPA'), 'modules')
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
if self.cfg['v2023']:
options['INCS'] += ' -I%s ' % elpa_inc_dir
else:
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
if LooseVersion(self.version) >= LooseVersion('6.1'):
elpa_ver = ''.join(get_software_version('ELPA').split('.')[:2])
options['DFLAGS'] += ' -D__ELPA=%s' % elpa_ver
elpa_inc_dir = os.path.join(elpa, 'include', 'elpa-%s' % get_software_version('ELPA'), 'elpa')
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
if self.cfg['v2023']:
options['INCS'] += ' -I%s ' % elpa_inc_dir
else:
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
else:
options['DFLAGS'] += ' -D__ELPA3'

# CUDA
cuda = get_software_root('CUDA')
if cuda:
options['DFLAGS'] += ' -D__ACC -D__DBCSR_ACC'
options['LIBS'] += ' -lcudart -lcublas -lcufft -lrt'
options['NVCC'] = ' nvcc'

if self.cfg['v2023']:
options['DFLAGS'] += ' -D__OFFLOAD_CUDA -D__DBCSR_ACC '
options['LIBS'] += ' -lcufft -lcudart -lnvrtc -lcuda -lcublas'
options['OFFLOAD_CC'] = 'nvcc'
options['OFFLOAD_FLAGS'] = "-O3 -g -w --std=c++11 $(DFLAGS) -Xcompiler='-fopenmp -Wall -Wextra -Werror'"
options['OFFLOAD_TARGET'] = 'cuda'
options['GPUVER'] = self.cfg['gpuver']
options['CXX'] = 'mpicxx'
options['CXXFLAGS'] = '-O3 -fopenmp -g -w --std=c++14 -fPIC $(DFLAGS) $(INCS)'
else:
options['DFLAGS'] += ' -D__ACC -D__DBCSR_ACC'
options['LIBS'] += ' -lcudart -lcublas -lcufft -lrt'
options['NVCC'] = ' nvcc'
# reset typearch
self.typearch = "Linux-x86-64-%s-cuda" % self.toolchain.name
# avoid group nesting
options['LIBS'] = options['LIBS'].replace('-Wl,--start-group', '').replace('-Wl,--end-group', '')

Expand Down Expand Up @@ -302,7 +329,7 @@ def configure_common(self):
# openmp introduces 2 major differences
# -automatic is default: -noautomatic -auto-scalar
# some mem-bandwidth optimisation
if self.cfg['type'] == 'psmp':
if self.cfg['type'] == 'psmp' or self.cfg['v2023']:
self.openmp = self.toolchain.get_flag('openmp')

# determine which opt flags to use
Expand Down Expand Up @@ -342,28 +369,29 @@ def configure_common(self):
cflags = os.getenv('CFLAGS')
fflags = os.getenv('FFLAGS')
fflags_lowopt = re.sub('-O[0-9]', '-O1', fflags)

options = {
'CC': os.getenv('MPICC'),
'CPP': '',
'AR': 'ar -r',
'FC': '%s %s' % (os.getenv('MPIF90'), self.openmp),
'LD': '%s %s' % (os.getenv('MPIF90'), self.openmp),
'AR': 'ar -r',
'CPPFLAGS': '',

'FPIC': self.fpic,
'DFLAGS': ' -D__parallel -D__BLACS -D__SCALAPACK -D__FFTSG %s' % self.cfg['extradflags'],
'INCS': '',
'CFLAGS': '-O3 -fopenmp -ftree-vectorize -march=native -fno-math-errno -fopenmp -std=c11 $(FPIC) $(DEBUG) '
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not the way to do it, do not hardcode compiler flags like this.
-march=native only works for GCC for instance.
The old code was correct apart from the missing INCS and DFLAGS.

'$(INCS) $(DFLAGS) %s ' %
self.cfg['extracflags'],
'DEBUG': self.debug,

'FCFLAGS': '$(FCFLAGS%s)' % optflags,
'FREE': '',
'FCFLAGS': '$(FCFLAGS%s) $(INCS)' % optflags,
'FCFLAGS2': '$(FCFLAGS%s)' % regflags,

'CFLAGS': ' %s %s %s $(FPIC) $(DEBUG) %s ' % (cflags, cppflags, ldflags, self.cfg['extracflags']),
'DFLAGS': ' -D__parallel -D__BLACS -D__SCALAPACK -D__FFTSG %s' % self.cfg['extradflags'],

'LIBS': os.getenv('LIBS', ''),

'FCFLAGSNOOPT': '$(DFLAGS) $(CFLAGS) -O0 $(FREE) $(FPIC) $(DEBUG)',
'FCFLAGSOPT': '%s $(FREE) $(SAFE) $(FPIC) $(DEBUG)' % fflags,
'FCFLAGSOPT2': '%s $(FREE) $(SAFE) $(FPIC) $(DEBUG)' % fflags_lowopt,

'LDFLAGS': '$(FCFLAGS) %s ' % ldflags,
'LIBS': os.getenv('LIBS', ''),

}

libint = get_software_root('LibInt')
Expand Down Expand Up @@ -410,7 +438,10 @@ def configure_common(self):
options['LIBS'] += ' %s -lstdc++ %s' % (libint_libs, libint_wrapper)

# add Libint include dir to $FCFLAGS
options['FCFLAGS'] += ' -I' + os.path.join(libint, 'include')
if self.cfg['v2023']:
options['INCS'] += ' -I' + os.path.join(libint, 'include')
else:
options['FCFLAGS'] += ' -I' + os.path.join(libint, 'include')

else:
# throw a warning, since CP2K without Libint doesn't make much sense
Expand All @@ -436,6 +467,10 @@ def configure_common(self):
options['LIBS'] += ' -L%s/lib -lxcf90 -lxc' % libxc
else:
options['LIBS'] += ' -L%s/lib -lxc' % libxc

if self.cfg['v2023']:
options['INCS'] += ' -I%s/include ' % libxc

self.log.info("Using Libxc-%s" % cur_libxc_version)
else:
self.log.info("libxc module not loaded, so building without libxc support")
Expand Down Expand Up @@ -488,7 +523,7 @@ def configure_intel_based(self):
# Required due to memory leak that occurs if high optimizations are used (from CP2K 7.1 intel-popt-makefile)
if ifortver >= LooseVersion("2018.5"):
self.make_instructions += "mp2_optimize_ri_basis.o: mp2_optimize_ri_basis.F\n" \
"\t$(FC) -c $(subst O2,O0,$(FCFLAGSOPT)) $<\n"
"\t$(FC) -c $(subst O2,O0,$(FCFLAGSOPT)) $<\n"
self.log.info("Optimization level of mp2_optimize_ri_basis.F was decreased to '-O0'")

# RHEL8 intel/2020a lots of CPASSERT failed (due to high optimization in cholesky decomposition)
Expand Down Expand Up @@ -534,20 +569,26 @@ def configure_intel_based(self):
def configure_GCC_based(self):
"""Configure for GCC based toolchains"""
options = self.configure_common()

options.update({
# need this to prevent "Unterminated character constant beginning" errors
'FREE': '-ffree-form -ffree-line-length-none',

'LDFLAGS': '$(FCFLAGS)',
'OBJECTS_ARCHITECTURE': 'machine_gfortran.o',
})
if self.cfg['v2023']:
options.update({
# need this to prevent "Unterminated character constant beginning" errors
'FREE': '-ffree-form -ffree-line-length-none -std=f2008',
})
options[
'FCFLAGSOPT'] = '-O3 -ftree-vectorize -march=native -fno-math-errno -fopenmp -fPIC $(FREE) $(DFLAGS) ' \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Again, don't hardcode compiler options

'-fmax-stack-var-size=32768'
else:
options.update({
# need this to prevent "Unterminated character constant beginning" errors
'FREE': '-ffree-form -ffree-line-length-none ',
'LDFLAGS': '$(FCFLAGS) ',
'OBJECTS_ARCHITECTURE': 'machine_gfortran.o',
})
options['FCFLAGSOPT'] += ' $(DFLAGS) $(CFLAGS) -fmax-stack-var-size=32768'
options['FCFLAGSOPT2'] += ' $(DFLAGS) $(CFLAGS)'

options['DFLAGS'] += ' -D__GFORTRAN'

options['FCFLAGSOPT'] += ' $(DFLAGS) $(CFLAGS) -fmax-stack-var-size=32768'
options['FCFLAGSOPT2'] += ' $(DFLAGS) $(CFLAGS)'

gcc_version = get_software_version('GCCcore') or get_software_version('GCC')
if LooseVersion(gcc_version) >= LooseVersion('10.0') and LooseVersion(self.version) <= LooseVersion('7.1'):
# -fallow-argument-mismatch is required for CP2K 7.1 (and older) when compiling with GCC 10.x & more recent,
Expand All @@ -573,12 +614,18 @@ def configure_ACML(self, options):
blas = os.getenv('LIBBLAS', '')
blas = blas.replace('gfortran64', 'gfortran64%s' % openmp_suffix)
options['LIBS'] += ' %s %s %s' % (self.libsmm, os.getenv('LIBSCALAPACK', ''), blas)
if self.cfg['v2023']:
options['LDFLAGS'] += ' -L%s/lib ' % self.libsmm_path
options['INCS'] += ' -I%s/include ' % self.libsmm_path

return options

def configure_BLAS_lib(self, options):
"""Configure for BLAS library."""
options['LIBS'] += ' %s %s' % (self.libsmm, os.getenv('LIBBLAS', ''))
if self.cfg['v2023']:
options['LDFLAGS'] += ' -L%s/lib ' % self.libsmm_path
options['INCS'] += ' -I%s/include ' % self.libsmm_path
return options

def configure_MKL(self, options):
Expand Down Expand Up @@ -614,24 +661,31 @@ def configure_MKL(self, options):
options['INTEL_INCF'] = '$(INTEL_INC)/fftw'
options['LIBS'] = '%s %s' % (os.getenv('LIBFFT', ''), options['LIBS'])

if self.cfg['v2023']:
options['LDFLAGS'] += ' %s ' % self.libsmm

return options

def configure_FFTW3(self, options):
"""Configure for FFTW3"""

options.update({
'FFTW_INC': os.getenv('FFT_INC_DIR', ''), # GCC
'FFTW3INC': os.getenv('FFT_INC_DIR', ''), # Intel
'FFTW3LIB': os.getenv('FFT_LIB_DIR', ''), # Intel
})

options['DFLAGS'] += ' -D__FFTW3'
if self.cfg['type'] == 'psmp':
if self.cfg['v2023']:
libfft = os.getenv('LIBFFT_MT', '')
options['LIBS'] += ' -lfftw3_omp -lfftw3'
options['LDFLAGS'] += ' -L%s ' % os.getenv('FFT_LIB_DIR', '')
options['INCS'] += ' -I%s ' % os.getenv('FFT_INC_DIR', '')
else:
libfft = os.getenv('LIBFFT', '')
options['LIBS'] += ' -L%s %s' % (os.getenv('FFT_LIB_DIR', '.'), libfft)
options.update({
'FFTW_INC': os.getenv('FFT_INC_DIR', ''), # GCC
'FFTW3INC': os.getenv('FFT_INC_DIR', ''), # Intel
'FFTW3LIB': os.getenv('FFT_LIB_DIR', ''), # Intel
})

if self.cfg['type'] == 'psmp':
libfft = os.getenv('LIBFFT_MT', '')
else:
libfft = os.getenv('LIBFFT', '')
options['LIBS'] += ' -L%s %s' % (os.getenv('FFT_LIB_DIR', '.'), libfft)
options['DFLAGS'] += ' -D__FFTW3'
return options

def configure_LAPACK(self, options):
Expand Down Expand Up @@ -764,14 +818,14 @@ def test_step(self):
'maxtasks=%(maxtasks)s',
'cp2k_run_prefix="%(mpicmd_prefix)s"',
]) % {
'f90': os.getenv('F90'),
'base': os.path.dirname(os.path.normpath(self.cfg['start_dir'])),
'cp2k_version': self.cfg['type'],
'triplet': self.typearch,
'cp2k_dir': os.path.basename(os.path.normpath(self.cfg['start_dir'])),
'maxtasks': self.cfg['maxtasks'],
'mpicmd_prefix': self.toolchain.mpi_cmd_for('', test_core_cnt),
}
'f90': os.getenv('F90'),
'base': os.path.dirname(os.path.normpath(self.cfg['start_dir'])),
'cp2k_version': self.cfg['type'],
'triplet': self.typearch,
'cp2k_dir': os.path.basename(os.path.normpath(self.cfg['start_dir'])),
'maxtasks': self.cfg['maxtasks'],
'mpicmd_prefix': self.toolchain.mpi_cmd_for('', test_core_cnt),
}
Comment on lines +821 to +828
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like an unintended shift-right


write_file(cfg_fn, cfg_txt)
self.log.debug("Contents of %s: %s" % (cfg_fn, cfg_txt))
Expand Down