Merge pull request #461 from LLNL/release/4.1.1

Release/4.1.1

.gitlab-ci.yml

@@ -32,20 +32,38 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
   BUILD_ROOT: ${CI_PROJECT_DIR}
+  PYTHON_ENVIRONMENT_PATH: ${CI_PROJECT_DIR}/.venv
 
 # Normally, stages are blocking in Gitlab. However, using the keyword "needs" we
 # can express dependencies between job that break the ordering of stages, in
 # favor of a DAG.
 # In practice q_*, l_* and b_* stages are independently run and start immediately.
 
 stages:
+  - environment
   - q_allocate_resources
   - q_build_and_test
   - q_release_resources
   - l_build_and_test
   - b_build_and_test
   - multi_project
 
+configure_python:
+  variables:
+    GIT_STRATEGY: none
+  tags:
+    - shell
+    - quartz
+  stage: environment
+  script:
+    - virtualenv -p /usr/tce/packages/python/python-3.8.2/bin/python ${PYTHON_ENVIRONMENT_PATH}
+    - . ${PYTHON_ENVIRONMENT_PATH}/bin/activate
+    - pip install lxml
+  cache:
+    key: venv
+    paths:
+      - ${PYTHON_ENVIRONMENT_PATH}
+
 # This is the rules that drives the activation of "advanced" jobs. All advanced
 # jobs will share this through a template mechanism.
 .advanced_pipeline:

.gitlab/quartz-templates.yml

@@ -30,7 +30,6 @@ allocate_resources (on quartz):
   stage: q_allocate_resources
   script:
     - salloc -N 1 -c 36 -p pdebug -t 20 --no-shell --job-name=${ALLOC_NAME}
-  needs: []
 
 ####
 # In post-build phase, deallocate resources
@@ -47,8 +46,13 @@ release_resources (on quartz):
 ####
 # Generic quartz build job, extending build script
 .build_and_test_on_quartz:
-  stage: q_build_and_test
   extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
+  stage: q_build_and_test
+  cache:
+    key: venv
+    paths:
+      - ${PYTHON_ENVIRONMENT_PATH}
+    policy: pull
 
 .build_and_test_on_quartz_advanced:
   extends: [.build_and_test_on_quartz, .advanced_pipeline]

CHANGELOG.md

@@ -16,6 +16,24 @@ Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to
 
 ### Fixed
 
+## [v4.1.1] - 2020-10-05
+
+### Added
+
+- Added primary pools test for allocation/deallocation overhead checking
+
+### Changed
+
+### Removed
+
+### Fixed
+
+- Fixed DynamicPoolMap deallocate to make coalesce check O(1) again.
+
+- Initialize m_default_allocator to HOST if not set explicitly.
+
+- Removed unreachable code that PGI compiler was giving compile warnings about.
+
 ## [v4.1.0] - 2020-09-28
 
 ### Added
@@ -33,6 +51,8 @@ Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to
 - GitLab test script now converts CTest output to JUnit so that test results
   are visible in the native GitLab UI.
 
+- Gitlab test scripts now caches python virtual environment.
+
 ### Removed
 
 - Peer access is no longer automatically enabled for CUDA and HIP.
@@ -54,6 +74,8 @@ Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to
 
 - Added cmake check to deterime if build subsystem capable of ASAN.
 
+- CI script junit generation
+
 ## [v4.0.1] - 2020-09-03
 
 ### Fixed

CMakeLists.txt

@@ -11,7 +11,7 @@ cmake_policy(SET CMP0057 NEW)
 
 project(Umpire
   LANGUAGES CXX C
-  VERSION 4.1.0)
+  VERSION 4.1.1)
 
 set(UMPIRE_VERSION_RC "")
 

Dockerfile

@@ -4,55 +4,55 @@ COPY --chown=axom:axom . /home/axom/workspace
 WORKDIR /home/axom/workspace
 RUN mkdir build && cd build && cmake -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_COMPILER=g++ ..
 RUN cd build && make -j 16
-RUN cd build && make test
+RUN cd build && ctest -T test --output-on-failure
 
 FROM axom/compilers:gcc-6 AS gcc6
 ENV GTEST_COLOR=1
 COPY --chown=axom:axom . /home/axom/workspace
 WORKDIR /home/axom/workspace
 RUN mkdir build && cd build && cmake -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_COMPILER=g++  ..
 RUN cd build && make -j 16
-RUN cd build && make test
+RUN cd build && ctest -T test --output-on-failure
 
 FROM axom/compilers:gcc-7 AS gcc7
 ENV GTEST_COLOR=1
 COPY --chown=axom:axom . /home/axom/workspace
 WORKDIR /home/axom/workspace
 RUN mkdir build && cd build && cmake -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_COMPILER=g++  ..
 RUN cd build && make -j 16
-RUN cd build && make test
+RUN cd build && ctest -T test --output-on-failure
 
 FROM axom/compilers:gcc-8 AS gcc
 ENV GTEST_COLOR=1
 COPY --chown=axom:axom . /home/axom/workspace
 WORKDIR /home/axom/workspace
 RUN mkdir build && cd build && cmake -DENABLE_C=On -DENABLE_COVERAGE=On -DCMAKE_BUILD_TYPE=Debug -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_COMPILER=g++  ..
 RUN cd build && make -j 16
-RUN cd build && make test
+RUN cd build && ctest -T test --output-on-failure
 
 FROM axom/compilers:clang-4 AS clang4
 ENV GTEST_COLOR=1
 COPY --chown=axom:axom . /home/axom/workspace
 WORKDIR /home/axom/workspace
 RUN mkdir build && cd build && cmake -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_COMPILER=clang++ ..
 RUN cd build && make -j 16
-RUN cd build && make test
+RUN cd build && ctest -T test --output-on-failure
 
 FROM axom/compilers:clang-5 AS clang5
 ENV GTEST_COLOR=1
 COPY --chown=axom:axom . /home/axom/workspace
 WORKDIR /home/axom/workspace
 RUN mkdir build && cd build && cmake -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_COMPILER=clang++ ..
 RUN cd build && make -j 16
-RUN cd build && make test
+RUN cd build && ctest -T test --output-on-failure
 
 FROM axom/compilers:clang-6 AS clang
 ENV GTEST_COLOR=1
 COPY --chown=axom:axom . /home/axom/workspace
 WORKDIR /home/axom/workspace
 RUN mkdir build && cd build && cmake -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_COMPILER=clang++ ..
 RUN cd build && make -j 16
-RUN cd build && make test
+RUN cd build && ctest -T test --output-on-failure
 
 FROM axom/compilers:nvcc-10 AS nvcc
 ENV GTEST_COLOR=1

README.md

@@ -1,4 +1,4 @@
-# <img src="https://cdn.rawgit.com/LLNL/Umpire/develop/share/umpire/logo/umpire-logo.png" width="128" valign="middle" alt="Umpire"/>  Umpire v4.1.0
+# <img src="https://cdn.rawgit.com/LLNL/Umpire/develop/share/umpire/logo/umpire-logo.png" width="128" valign="middle" alt="Umpire"/>  Umpire v4.1.1
 
 [![Travis Build Status](https://travis-ci.com/LLNL/Umpire.svg?branch=develop)](https://travis-ci.com/LLNL/Umpire)
 [![Azure Pipelines Build Status](https://dev.azure.com/davidbeckingsale/Umpire/_apis/build/status/LLNL.Umpire?branchName=develop)](https://dev.azure.com/davidbeckingsale/Umpire/_build/latest?definitionId=1&branchName=develop)

RELEASE_NOTES.md

@@ -1,3 +1,9 @@
+# v4.1.1
+
+- Fixed DynamicPoolMap deallocate to make coalesce check O(1) again.
+
+- Initialize m_default_allocator to HOST if not set explicitly.
+
 # v4.1.0
 
 - QuickPool available via the C & Fortran APIs.

docs/sphinx/conf.py

@@ -98,7 +98,7 @@
 # The short X.Y version.
 version = u'4.1'
 # The full version, including alpha/beta/rc tags.
-release = u'4.1.0'
+release = u'4.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/sphinx/conf.py.in

@@ -98,7 +98,7 @@ author = u'David Beckingsale'
 # The short X.Y version.
 version = u'4.1'
 # The full version, including alpha/beta/rc tags.
-release = u'4.1.0'
+release = u'4.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

scripts/gitlab/build_and_test.sh

@@ -19,6 +19,9 @@ build_root=${BUILD_ROOT:-""}
 hostconfig=${HOST_CONFIG:-""}
 spec=${SPEC:-""}
 
+sys_type=${SYS_TYPE:-""}
+py_env_path=${PYTHON_ENVIRONMENT_PATH:-""}
+
 # Dependencies
 if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]]
 then
@@ -131,10 +134,16 @@ then
     cp Testing/*/Test.xml ${project_dir}
 
     # Convert CTest xml to JUnit (on toss3 only)
-    if [[ $SYS_TYPE == *toss3* ]]; then
-        ${project_dir}/scripts/gitlab/convert_to_junit.py \
-        ${project_dir}/Test.xml \
-        ${project_dir}/scripts/gitlab/junit.xslt > ${project_dir}/junit.xml
+    if [[ ${sys_type} == *toss_3* ]]; then
+        if [[ -n ${py_env_path} ]]; then
+            . ${py_env_path}/bin/activate
+
+            python3 ${project_dir}/scripts/gitlab/convert_to_junit.py \
+            ${project_dir}/Test.xml \
+            ${project_dir}/scripts/gitlab/junit.xslt > ${project_dir}/junit.xml
+        else
+            echo "ERROR: needs python env with lxml, please set PYTHON_ENVIRONMENT_PATH"
+        fi
     fi
 
     if grep -q "Errors while running CTest" ./tests_output.txt

scripts/make_release_tarball.sh

@@ -7,7 +7,7 @@
 ##############################################################################
 
 TAR_CMD=gtar
-VERSION=4.1.0
+VERSION=4.1.1
 
 git archive --prefix=umpire-${VERSION}/ -o umpire-${VERSION}.tar HEAD 2> /dev/null
 

src/umpire/ResourceManager.cpp

@@ -341,7 +341,6 @@ Allocator ResourceManager::makeResource(const std::string& name, MemoryResourceT
     if (name.find("::") == std::string::npos) {
       m_memory_resources[resource::string_to_resource(name)] = allocator.get();
     }
-    m_default_allocator = allocator.get();
     m_allocators_by_id[id] = allocator.get();
     m_allocators.emplace_front(std::move(allocator));
 
@@ -414,8 +413,8 @@ Allocator ResourceManager::getDefaultAllocator()
   UMPIRE_LOG(Debug, "");
 
   if (!m_default_allocator) {
-    return getAllocator("HOST");
-    UMPIRE_ERROR("The default Allocator is not defined");
+    UMPIRE_LOG(Debug, "Initializing m_default_allocator as HOST");
+    m_default_allocator = getAllocator("HOST").getAllocationStrategy();
   }
 
   return Allocator(m_default_allocator);
@@ -623,7 +622,7 @@ void* ResourceManager::reallocate(void* current_ptr, std::size_t new_size)
     auto alloc_record = m_allocations.find(current_ptr);
     strategy = alloc_record->strategy;
   } else {
-    strategy = m_default_allocator;
+    strategy = getDefaultAllocator().getAllocationStrategy();
   }
 
   void* new_ptr{reallocate_impl(current_ptr, new_size, Allocator(strategy))};

src/umpire/resource/MemoryResourceTypes.hpp

@@ -40,9 +40,8 @@ inline std::string resource_to_string(MemoryResourceType type)
       return "DEVICE_CONST";
     case File:
       return "FILE";
-    default: 
+    default:
       UMPIRE_ERROR("Unkown resource type: " << type);
-      return "UNKNOWN"; 
   }
 }
 
@@ -56,14 +55,13 @@ inline MemoryResourceType string_to_resource(const std::string& resource)
   else if (resource == "FILE") return MemoryResourceType::File;
   else {
     UMPIRE_ERROR("Unkown resource name: " << resource);
-    return MemoryResourceType::Unknown;
   }
 }
 
 inline int resource_to_device_id(const std::string& resource) {
   int device_id{0};
   if (resource.find("::") != std::string::npos) {
-    device_id = std::stoi(resource.substr(resource.find("::") + 2)); 
+    device_id = std::stoi(resource.substr(resource.find("::") + 2));
   }
   return device_id;
 }

src/umpire/strategy/DynamicPoolList.cpp

@@ -55,6 +55,13 @@ void DynamicPoolList::release()
   dpa.release();
 }
 
+std::size_t DynamicPoolList::getCurrentSize() const noexcept
+{
+  std::size_t CurrentSize = dpa.getCurrentSize();
+  UMPIRE_LOG(Debug, "() returning " << CurrentSize);
+  return CurrentSize;
+}
+
 std::size_t DynamicPoolList::getActualSize() const noexcept
 {
   std::size_t ActualSize = dpa.getActualSize();
@@ -115,7 +122,7 @@ DynamicPoolList::CoalesceHeuristic DynamicPoolList::percent_releasable(
         [=](const DynamicPoolList& UMPIRE_UNUSED_ARG(pool)) { return false; };
   } else if (percentage == 100) {
     return [=](const strategy::DynamicPoolList& pool) {
-      return (pool.getActualSize() == pool.getReleasableSize());
+      return ( pool.getCurrentSize() == 0 );
     };
   } else {
     float f = (float)((float)percentage / (float)100.0);

src/umpire/strategy/DynamicPoolList.hpp

@@ -65,6 +65,7 @@ class DynamicPoolList : public AllocationStrategy {
   void release() override;
 
   std::size_t getActualSize() const noexcept override;
+  std::size_t getCurrentSize() const noexcept override;
 
   Platform getPlatform() noexcept override;
 

src/umpire/strategy/DynamicPoolMap.cpp

@@ -122,6 +122,8 @@ void* DynamicPoolMap::allocate(std::size_t bytes)
                  alloc_bytes);
   }
 
+  m_current_bytes += bytes;
+
   UMPIRE_UNPOISON_MEMORY_REGION(m_allocator, ptr, bytes);
   return ptr;
 }
@@ -140,6 +142,8 @@ void DynamicPoolMap::deallocate(void* ptr)
     std::size_t whole_bytes;
     std::tie(bytes, is_head, whole_bytes) = *iter->second;
 
+    m_current_bytes -= bytes;
+
     // Insert in free map
     insertFree(ptr, bytes, is_head, whole_bytes);
 
@@ -177,6 +181,12 @@ std::size_t DynamicPoolMap::getActualSize() const noexcept
   return m_actual_bytes;
 }
 
+std::size_t DynamicPoolMap::getCurrentSize() const noexcept
+{
+  UMPIRE_LOG(Debug, "() returning " << m_current_bytes);
+  return m_current_bytes;
+}
+
 std::size_t DynamicPoolMap::getFreeBlocks() const noexcept
 {
   return m_free_map.size();
@@ -473,7 +483,7 @@ DynamicPoolMap::CoalesceHeuristic DynamicPoolMap::percent_releasable(
     return [=](const DynamicPoolMap& UMPIRE_UNUSED_ARG(pool)) { return false; };
   } else if (percentage == 100) {
     return [=](const strategy::DynamicPoolMap& pool) {
-      return (pool.getActualSize() == pool.getReleasableSize());
+      return ( pool.getCurrentSize() == 0 );
     };
   } else {
     float f = (float)((float)percentage / (float)100.0);

src/umpire/strategy/DynamicPoolMap.hpp

@@ -71,6 +71,7 @@ class DynamicPoolMap : public AllocationStrategy,
   void release() override;
 
   std::size_t getActualSize() const noexcept override;
+  std::size_t getCurrentSize() const noexcept override;
 
   Platform getPlatform() noexcept override;
 
@@ -178,6 +179,7 @@ class DynamicPoolMap : public AllocationStrategy,
   const std::size_t m_next_minimum_pool_allocation_size;
 
   std::size_t m_actual_bytes{0};
+  std::size_t m_current_bytes{0};
   bool m_is_destructing{false};
 };