Added support for OpenACC

.testfiles/jacobi.c

@@ -0,0 +1,85 @@
+/*
+ *  Copyright 2013 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <math.h>
+#include <string.h>
+#include <openacc.h>
+#include <stdio.h>
+#include "timer.h"
+
+#define NN 1024
+#define NM 1024
+
+double A[NN][NM];
+double Anew[NN][NM];
+
+int main(int argc, char** argv)
+{
+    const int n = NN;
+    const int m = NM;
+    const int iter_max = 1000;
+
+    const double tol = 1.0e-6;
+    double error     = 1.0;
+
+    memset(A, 0, n * m * sizeof(double));
+    memset(Anew, 0, n * m * sizeof(double));
+
+    for (int j = 0; j < n; j++)
+    {
+        A[j][0]    = 1.0;
+        Anew[j][0] = 1.0;
+    }
+
+    printf("Jacobi relaxation Calculation: %d x %d mesh\n", n, m);
+
+    StartTimer();
+    int iter = 0;
+
+#pragma acc data copyin(Anew), copy(A) 
+    while ( error > tol && iter < iter_max )
+    {
+        error = 0.0;
+
+#pragma acc kernels
+        for( int j = 1; j < n-1; j++)
+        {
+            for( int i = 1; i < m-1; i++ )
+            {
+                Anew[j][i] = 0.25 * ( A[j][i+1] + A[j][i-1]
+                                    + A[j-1][i] + A[j+1][i]);
+                error = fmax( error, fabs(Anew[j][i] - A[j][i]));
+            }
+        }
+
+#pragma acc kernels
+        for( int j = 1; j < n-1; j++)
+        {
+            for( int i = 1; i < m-1; i++ )
+            {
+                A[j][i] = Anew[j][i];    
+            }
+        }
+
+        if(iter % 100 == 0) printf("%5d, %0.6f\n", iter, error);
+
+        iter++;
+    }
+
+    double runtime = GetTimer();
+
+    printf(" total: %f s\n", runtime / 1000);
+}

.testfiles/timer.h

@@ -0,0 +1,67 @@
+/*
+ *  Copyright 2012 NVIDIA Corporation
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef TIMER_H
+#define TIMER_H
+
+#include <stdlib.h>
+
+#ifdef WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#else
+#include <sys/time.h>
+#endif
+
+#ifdef WIN32
+double PCFreq = 0.0;
+__int64 timerStart = 0;
+#else
+struct timeval timerStart;
+#endif
+
+void StartTimer()
+{
+#ifdef WIN32
+    LARGE_INTEGER li;
+    if(!QueryPerformanceFrequency(&li))
+        printf("QueryPerformanceFrequency failed!\n");
+
+    PCFreq = (double)li.QuadPart/1000.0;
+
+    QueryPerformanceCounter(&li);
+    timerStart = li.QuadPart;
+#else
+    gettimeofday(&timerStart, NULL);
+#endif
+}
+
+// time elapsed in ms
+double GetTimer()
+{
+#ifdef WIN32
+    LARGE_INTEGER li;
+    QueryPerformanceCounter(&li);
+    return (double)(li.QuadPart-timerStart)/PCFreq;
+#else
+    struct timeval timerStop, timerElapsed;
+    gettimeofday(&timerStop, NULL);
+    timersub(&timerStop, &timerStart, &timerElapsed);
+    return timerElapsed.tv_sec*1000.0+timerElapsed.tv_usec/1000.0;
+#endif
+}
+
+#endif // TIMER_H

docs/design.rst

@@ -8,7 +8,7 @@ General Definitions
   A list of configuration options (a "configuration object") that completely describes the hardware and
   software environment in which the TAU Commander workflow will be performed.  This includes, among other things, the
   operating system distribution (e.g. Linux, Darwin, CNL, etc.) CPU architecture (x86_64, ppc64, etc.) compiler
-  installation paths, compiler family (e.g. Intel, Cray, PGI, etc.), MPI installation path, and TAU and PDT
+  installation paths, compiler family (e.g. Intel, Cray, PGI, NVHPC, etc.), MPI installation path, and TAU and PDT
   installation paths.
 
 :Application:

packages/taucmdr/cf/compiler/cuda.py

@@ -33,6 +33,8 @@
                                CXX=('CUDA', 'CU'),
                                FC=('CUDA Fortran', 'CUF'))
 
+NVHPC = CUDA_COMPILERS.add('NVHPC', CXX='nvcc')
+
 NVIDIA = CUDA_COMPILERS.add('NVIDIA', CXX='nvcc')
 
 IBM = CUDA_COMPILERS.add('IBM', family_regex=r'^IBM XL', version_flags=['-qversion'],

packages/taucmdr/cf/compiler/host.py

@@ -48,6 +48,9 @@
 PGI = HOST_COMPILERS.add('PGI', family_regex=r'The Portland Group|NVIDIA CORPORATION',
                          CC='pgcc', CXX=('pgCC', 'pgc++', 'pgcxx'), FC=('pgfortran', 'pgf90', 'pgf77'))
 
+NVHPC = HOST_COMPILERS.add('NVHPC', family_regex=r'NVIDIA CORPORATION',
+                         CC='nvc' , CXX=('nvc++', 'nvcc'), FC='nvfortran')
+
 IBM = HOST_COMPILERS.add('IBM', family_regex=r'^IBM XL',
                          version_flags=['-qversion'],
                          CC=('xlc_r', 'xlc'),

packages/taucmdr/cf/compiler/mpi.py

@@ -42,6 +42,11 @@
                               CXX=('MPI C++', 'MPI_CXX'),
                               FC=('MPI Fortran', ('MPI_FC', 'MPI_F77', 'MPI_F90')))
 
+NVHPC = MPI_COMPILERS.add('NVHPC', family_regex=r'NVIDIA CORPORATION', show_wrapper_flags=['-show'],
+                           CC='mpicc',
+                           CXX=('mpic++', 'mpicxx', 'mpiCC'),
+                           FC=('mpiftn', 'mpif90', 'mpif77', 'mpifort'))
+
 SYSTEM = MPI_COMPILERS.add('System', show_wrapper_flags=['-show'],
                            CC='mpicc',
                            CXX=('mpic++', 'mpicxx', 'mpiCC'),

packages/taucmdr/cf/software/binutils_installation.py

@@ -39,7 +39,7 @@
 from taucmdr.error import ConfigurationError
 from taucmdr.cf.software import SoftwarePackageError
 from taucmdr.cf.software.installation import AutotoolsInstallation
-from taucmdr.cf.compiler.host import CC, CXX, PGI, GNU
+from taucmdr.cf.compiler.host import CC, CXX, PGI, NVHPC, GNU
 
 
 LOGGER = logger.get_logger(__name__)
@@ -54,8 +54,8 @@ class BinutilsInstallation(AutotoolsInstallation):
     """Encapsulates a GNU binutils installation."""
 
     def __init__(self, sources, target_arch, target_os, compilers):
-        # binutils can't be built with PGI compilers so substitute GNU compilers instead
-        if compilers[CC].unwrap().info.family is PGI:
+        # binutils can't be built with PGI or NVHPC compilers so substitute GNU compilers instead
+        if compilers[CC].unwrap().info.family is PGI or compilers[CC].unwrap().info.family is NVHPC:
             try:
                 gnu_compilers = GNU.installation()
             except ConfigurationError as err:

packages/taucmdr/cf/software/installation.py

@@ -565,7 +565,7 @@ class AutotoolsInstallation(MakeInstallation):
         make [flags] install [options]
     """
 
-    def configure(self, flags):
+    def configure(self, flags, env=None):
         """Invoke `configure`.
 
         Args:
@@ -578,7 +578,7 @@ def configure(self, flags):
         LOGGER.debug("Configuring %s at '%s'", self.name, self._src_prefix)
         cmd = ['./configure', '--prefix=%s' % self.install_prefix] + flags
         LOGGER.info("Configuring %s...", self.title)
-        if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
+        if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True, env=env):
             util.add_error_stack(self._src_prefix)
             raise SoftwarePackageError('%s configure failed' % self.title)
 

packages/taucmdr/cf/software/libelf_installation.py

@@ -31,7 +31,11 @@
             an architecture-independent way.
 """
 
+import os
 from taucmdr.cf.software.installation import AutotoolsInstallation
+from taucmdr.error import ConfigurationError
+from taucmdr.cf.software import SoftwarePackageError
+from taucmdr.cf.compiler.host import CC, CXX, PGI, GNU, NVHPC
 
 REPOS = {None: ['http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/elfutils-0.180.tar.bz2',
                 'https://sourceware.org/elfutils/ftp/0.180/elfutils-0.180.tar.bz2']}
@@ -45,9 +49,16 @@ class LibelfInstallation(AutotoolsInstallation):
     """Encapsulates a libelf installation."""
 
     def __init__(self, sources, target_arch, target_os, compilers):
+        if compilers[CC].unwrap().info.family is PGI or compilers[CC].unwrap().info.family is NVHPC :
+            try:
+                gnu_compilers = GNU.installation()
+            except ConfigurationError as err:
+                raise SoftwarePackageError("GNU compilers (required to build libelf) could not be found.") from err
+            compilers = compilers.modify(Host_CC=gnu_compilers[CC], Host_CXX=gnu_compilers[CXX])
         super().__init__('libelf', 'libelf', sources,
                                                  target_arch, target_os, compilers, REPOS, None, LIBRARIES, HEADERS)
 
     def configure(self, flags):
         flags.extend(['--disable-debuginfod'])
-        return super().configure(flags)
+        # Libelf's configure script requires CC to be a GNU compiler, so we have to set the environment variable before running the configure script
+        super().configure(flags, env={'CC':GNU.installation()[CC].unwrap().absolute_path})

packages/taucmdr/cf/software/libunwind_installation.py

@@ -39,7 +39,7 @@
 from taucmdr.cf.platforms import IBM_BGQ, ARM64, PPC64LE, PPC64, CRAY_CNL, LINUX
 from taucmdr.cf.software import SoftwarePackageError
 from taucmdr.cf.software.installation import AutotoolsInstallation
-from taucmdr.cf.compiler.host import CC, CXX, PGI, GNU
+from taucmdr.cf.compiler.host import CC, CXX, PGI, GNU, NVHPC
 
 
 
@@ -60,8 +60,8 @@ class LibunwindInstallation(AutotoolsInstallation):
     """Encapsulates a libunwind installation."""
 
     def __init__(self, sources, target_arch, target_os, compilers):
-        # libunwind can't be built with PGI compilers so substitute GNU compilers instead
-        if compilers[CC].unwrap().info.family is PGI:
+        # libunwind can't be built with PGI or NVHPC compilers so substitute GNU compilers instead
+        if compilers[CC].unwrap().info.family is PGI or compilers[CC].unwrap().info.family is NVHPC :
             try:
                 gnu_compilers = GNU.installation()
             except ConfigurationError as err:

packages/taucmdr/cf/software/papi_installation.py

@@ -40,7 +40,7 @@
 from taucmdr.error import ConfigurationError
 from taucmdr.cf.software import SoftwarePackageError
 from taucmdr.cf.software.installation import AutotoolsInstallation
-from taucmdr.cf.compiler.host import CC, CXX, IBM, GNU
+from taucmdr.cf.compiler.host import CC, CXX, IBM, GNU, NVHPC
 
 LOGGER = logger.get_logger(__name__)
 
@@ -54,8 +54,8 @@ class PapiInstallation(AutotoolsInstallation):
     """Encapsulates a PAPI installation."""
 
     def __init__(self, sources, target_arch, target_os, compilers):
-        # PAPI can't be built with IBM compilers so substitute GNU compilers instead
-        if compilers[CC].unwrap().info.family is IBM:
+        # PAPI can't be built with IBM or NVHPC compilers so substitute GNU compilers instead
+        if compilers[CC].unwrap().info.family is IBM or compilers[CC].unwrap().info.family is NVHPC:
             try:
                 gnu_compilers = GNU.installation()
             except ConfigurationError as err:
@@ -70,6 +70,12 @@ def _prepare_src(self, *args, **kwargs):
         src_prefix = super()._prepare_src(*args, **kwargs)
         if os.path.basename(src_prefix) != 'src':
             src_prefix = os.path.join(src_prefix, 'src')
+
+        # Modify PAPI's config.mk script to not use -Werror due to upstream bug when using NVHPC
+        for line in fileinput.input(os.path.join(src_prefix, 'libpfm4/config.mk'), inplace=True):
+            # fileinput.input with inplace=1 redirects stdout to the input file ... freaky
+            sys.stdout.write(line.replace('-Werror',''))
+
         return src_prefix
 
     def configure(self, flags):

packages/taucmdr/cf/software/pdt_installation.py

@@ -36,7 +36,7 @@
 from taucmdr.cf.software.installation import AutotoolsInstallation
 from taucmdr.cf.platforms import TauMagic, X86_64, INTEL_KNL, IBM_BGQ, PPC64LE, LINUX, DARWIN, IBM_CNK,\
     ARM64
-from taucmdr.cf.compiler.host import CC, CXX, PGI, GNU, INTEL
+from taucmdr.cf.compiler.host import CC, CXX, PGI, NVHPC, GNU, INTEL
 
 
 LOGGER = logger.get_logger(__name__)
@@ -183,8 +183,8 @@ class PdtInstallation(AutotoolsInstallation):
     """
 
     def __init__(self, sources, target_arch, target_os, compilers):
-        # PDT 3.22 can't be built with PGI compilers so substitute GNU compilers instead
-        if compilers[CC].unwrap().info.family is PGI:
+        # PDT 3.22 can't be built with PGI or NVHPC compilers so substitute GNU compilers instead
+        if compilers[CC].unwrap().info.family is PGI or compilers[CC].unwrap().info.family is NVHPC :
             try:
                 gnu_compilers = GNU.installation()
             except ConfigurationError as err:
@@ -222,7 +222,7 @@ def verify(self):
             self.verify()
 
     def configure(self, _):
-        family_flags = {GNU.name: '-GNU', INTEL.name: '-icpc', PGI.name: '-pgCC'}
+        family_flags = {GNU.name: '-GNU', INTEL.name: '-icpc', PGI.name: '-pgCC', NVHPC.name: '-nvc++'}
         compiler_flag = family_flags.get(self.compilers[CXX].info.family.name, '')
         cmd = ['./configure', '-prefix=' + self.install_prefix, compiler_flag]
         LOGGER.info("Configuring PDT...")