From b0e510e61c17cdd27ee1ebc17eb29c38655a9cae Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Thu, 15 Apr 2021 08:20:52 -0700 Subject: [PATCH 1/6] Add runtime environment check for NERSC Cray hugepages --- Corrfunc/__init__.py | 2 ++ Corrfunc/utils.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/Corrfunc/__init__.py b/Corrfunc/__init__.py index 32a7d488..86e37f8a 100644 --- a/Corrfunc/__init__.py +++ b/Corrfunc/__init__.py @@ -24,6 +24,8 @@ from . import utils from . import theory from . import mocks + + utils.check_runtime_env() def read_text_file(filename, encoding="utf-8"): diff --git a/Corrfunc/utils.py b/Corrfunc/utils.py index d6191f59..493ffce2 100644 --- a/Corrfunc/utils.py +++ b/Corrfunc/utils.py @@ -6,7 +6,10 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) import sys +import os +import warnings from os.path import exists as file_exists + import wurlitzer from contextlib import contextmanager @@ -1053,6 +1056,19 @@ def sys_pipes(): yield except: yield + +def check_runtime_env(): + ''' + Detect any computing environment conditions that may cause Corrfunc + to fail, and inform the user if there is any action they can take. + ''' + + # Check if Cray hugepages is enabled at NERSC, which will crash + # C Python extensions due to a hugepages bug + if 'NERSC_HOST' in os.environ and os.getenv('HUGETLB_DEFAULT_PAGE_SIZE'): + warnings.warn('Warning: Cray hugepages has a bug that may crash Corrfunc. ' + 'You might be able to fix such a crash with `module unload craype-hugepages2M` ' + '(see https://github.com/manodeep/Corrfunc/issues/245 for details)') if __name__ == '__main__': import doctest From 228c300fc91cdd551e1cb43953833250d7f1f8c3 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Thu, 15 Apr 2021 16:21:37 -0400 Subject: [PATCH 2/6] Update ci.yml --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df61881a..6483c91b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-16.04", "ubuntu-18.04", "ubuntu-20.04", "macos-latest"] - compiler: [gcc-7, gcc-8, gcc-9, clang] + compiler: [gcc-7, gcc-9, gcc-10, clang] python-version: ["2.7", "3.5", "3.6", "3.7", "3.8"] numpy-version: ["1.14", "1.16", "1.18"] exclude: @@ -18,30 +18,30 @@ jobs: # Only run with 'clang' on OSX - os: "macos-latest" compiler: gcc-7 - - os: "macos-latest" - compiler: gcc-8 - os: "macos-latest" compiler: gcc-9 + - os: "macos-latest" + compiler: gcc-10 # Don't use 'clang' on linux + - os: "ubuntu-16.04" + compiler: clang - os: "ubuntu-18.04" compiler: clang - os: "ubuntu-20.04" compiler: clang - - os: "ubuntu-16.04" - compiler: clang - # only gcc-9 on 20.04 + # only gcc-10 on 20.04 - os: "ubuntu-20.04" - compiler: gcc-7 + compiler: gcc-9 - os: "ubuntu-20.04" - compiler: gcc-8 + compiler: gcc-7 # only gcc-7 on 16.04 - - os: "ubuntu-16.04" - compiler: gcc-8 - os: "ubuntu-16.04" compiler: gcc-9 + - os: "ubuntu-16.04" + compiler: gcc-10 # python3.8 only on 20.04 - os: "ubuntu-16.04" From f4931fd400d9680eaab91d89261e2f30e51010ef Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Thu, 15 Apr 2021 16:34:28 -0400 Subject: [PATCH 3/6] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6483c91b..f6bd3ffa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,9 +37,9 @@ jobs: - os: "ubuntu-20.04" compiler: gcc-7 - # only gcc-7 on 16.04 + # only gcc-9 on 16.04 - os: "ubuntu-16.04" - compiler: gcc-9 + compiler: gcc-7 - os: "ubuntu-16.04" compiler: gcc-10 From aef0e7a4d9b136ba920d7c15bfcfec1e79344b36 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Thu, 15 Apr 2021 16:52:18 -0400 Subject: [PATCH 4/6] Update changelog [ci skip] --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index c39bb308..91b09a51 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,7 @@ Enhancements ------------ - In the theoretical VPF calculation (``theory.vpf``), the total volume of the random spheres can now exceed the volume of the sample [#238] - Gridlink (the binning of particles into cells) now uses a parallel algorithm for the theory module [#239] +- Add detection of known-bad Cray hugepages library at NERSC [#246] Bug fixes --------- From aff9260fcc6bcada430f7651026928266c1f2d9f Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Thu, 15 Apr 2021 14:43:29 -0700 Subject: [PATCH 5/6] Fix pep8 whitespace --- Corrfunc/__init__.py | 2 +- Corrfunc/utils.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Corrfunc/__init__.py b/Corrfunc/__init__.py index 86e37f8a..78197ea0 100644 --- a/Corrfunc/__init__.py +++ b/Corrfunc/__init__.py @@ -24,7 +24,7 @@ from . import utils from . import theory from . import mocks - + utils.check_runtime_env() diff --git a/Corrfunc/utils.py b/Corrfunc/utils.py index 493ffce2..91cdec69 100644 --- a/Corrfunc/utils.py +++ b/Corrfunc/utils.py @@ -1056,19 +1056,22 @@ def sys_pipes(): yield except: yield - + + def check_runtime_env(): ''' Detect any computing environment conditions that may cause Corrfunc to fail, and inform the user if there is any action they can take. ''' - + # Check if Cray hugepages is enabled at NERSC, which will crash # C Python extensions due to a hugepages bug if 'NERSC_HOST' in os.environ and os.getenv('HUGETLB_DEFAULT_PAGE_SIZE'): - warnings.warn('Warning: Cray hugepages has a bug that may crash Corrfunc. ' - 'You might be able to fix such a crash with `module unload craype-hugepages2M` ' - '(see https://github.com/manodeep/Corrfunc/issues/245 for details)') + warnings.warn('Warning: Cray hugepages has a bug that may crash + 'Corrfunc. You might be able to fix such a crash with ' + '`module unload craype-hugepages2M` (see ' + 'https://github.com/manodeep/Corrfunc/issues/245 ' + 'for details)') if __name__ == '__main__': import doctest From 9e1737fa535aba502b3c03c14e6d9e98edeebc32 Mon Sep 17 00:00:00 2001 From: Lehman Garrison Date: Thu, 15 Apr 2021 14:54:23 -0700 Subject: [PATCH 6/6] Missing quote --- Corrfunc/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Corrfunc/utils.py b/Corrfunc/utils.py index 91cdec69..3e3d3979 100644 --- a/Corrfunc/utils.py +++ b/Corrfunc/utils.py @@ -1067,7 +1067,7 @@ def check_runtime_env(): # Check if Cray hugepages is enabled at NERSC, which will crash # C Python extensions due to a hugepages bug if 'NERSC_HOST' in os.environ and os.getenv('HUGETLB_DEFAULT_PAGE_SIZE'): - warnings.warn('Warning: Cray hugepages has a bug that may crash + warnings.warn('Warning: Cray hugepages has a bug that may crash ' 'Corrfunc. You might be able to fix such a crash with ' '`module unload craype-hugepages2M` (see ' 'https://github.com/manodeep/Corrfunc/issues/245 '