From 9c256b12b89f86b30fe1a6f03dd79db351d56fb3 Mon Sep 17 00:00:00 2001
From: Andrew Myers <atmyers2@gmail.com>
Date: Thu, 29 Jun 2023 16:24:53 -0700
Subject: [PATCH] Allow users to change the default vector growth stategy
 (#3389)

This allows users to set at runtime an alternate to the default vector
growth factor of 1.5. This could help save memory in simulations that
use a lot of particles and are close to the capacity of the GPU device.

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX
users
- [ ] include documentation in the code and/or rst files, if appropriate

---------

Co-authored-by: Weiqun Zhang <WeiqunZhang@lbl.gov>
---
 Src/Base/AMReX.cpp           |  1 +
 Src/Base/AMReX_PODVector.H   | 11 ++++++++++-
 Src/Base/AMReX_PODVector.cpp | 33 +++++++++++++++++++++++++++++++++
 Src/Base/CMakeLists.txt      |  1 +
 Src/Base/Make.package        |  1 +
 5 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 Src/Base/AMReX_PODVector.cpp

diff --git a/Src/Base/AMReX.cpp b/Src/Base/AMReX.cpp
index f52f698ac15..1d98f091c8d 100644
--- a/Src/Base/AMReX.cpp
+++ b/Src/Base/AMReX.cpp
@@ -604,6 +604,7 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
     iMultiFab::Initialize();
     VisMF::Initialize();
     AsyncOut::Initialize();
+    VectorGrowthStrategy::Initialize();
 
 #ifdef AMREX_USE_EB
     EB2::Initialize();
diff --git a/Src/Base/AMReX_PODVector.H b/Src/Base/AMReX_PODVector.H
index c0da235c30a..435ac287752 100644
--- a/Src/Base/AMReX_PODVector.H
+++ b/Src/Base/AMReX_PODVector.H
@@ -218,6 +218,14 @@ namespace amrex
         }
     }
 
+    namespace VectorGrowthStrategy
+    {
+        extern AMREX_EXPORT Real growth_factor;
+        inline Real GetGrowthFactor () { return growth_factor; }
+
+        void Initialize ();
+    }
+
     template <class T, class Allocator = std::allocator<T> >
     class PODVector : public Allocator
     {
@@ -612,7 +620,8 @@ namespace amrex
 
             while (new_capacity < (capacity() + a_num_to_be_added))
             {
-                new_capacity = (3 * new_capacity + 1)/2;
+                new_capacity = static_cast<size_type>(
+                    VectorGrowthStrategy::GetGrowthFactor() * static_cast<Real>(new_capacity + 1));
             }
 
             return new_capacity;
diff --git a/Src/Base/AMReX_PODVector.cpp b/Src/Base/AMReX_PODVector.cpp
new file mode 100644
index 00000000000..07952caa5fc
--- /dev/null
+++ b/Src/Base/AMReX_PODVector.cpp
@@ -0,0 +1,33 @@
+#include <AMReX_PODVector.H>
+#include <AMReX_ParmParse.H>
+#include <AMReX_REAL.H>
+
+namespace amrex::VectorGrowthStrategy
+{
+    Real growth_factor = 1.5_rt;
+
+    void Initialize () {
+        ParmParse pp("amrex");
+        pp.queryAdd("vector_growth_factor", growth_factor);
+
+        // clamp user input to reasonable values
+        constexpr Real min_factor = 1.05_rt;
+        constexpr Real max_factor = 4._rt;
+
+        if (growth_factor < min_factor) {
+            if (Verbose()) {
+                amrex::Print() << "Warning: user-provided vector growth factor is too small."
+                               << " Clamping to " << min_factor << ". \n";
+            }
+            growth_factor = min_factor;
+        }
+
+        if (growth_factor > max_factor) {
+            if (Verbose()) {
+                amrex::Print() << "Warning: user-provided vector growth factor is too large."
+                               << " Clamping to " << max_factor << ". \n";
+            }
+            growth_factor = max_factor;
+        }
+    }
+}
diff --git a/Src/Base/CMakeLists.txt b/Src/Base/CMakeLists.txt
index 42952dbc853..227da932626 100644
--- a/Src/Base/CMakeLists.txt
+++ b/Src/Base/CMakeLists.txt
@@ -25,6 +25,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
        AMReX_Exception.H
        AMReX_Extension.H
        AMReX_PODVector.H
+       AMReX_PODVector.cpp
        AMReX_ParmParse.cpp
        AMReX_parmparse_fi.cpp
        AMReX_ParmParse.H
diff --git a/Src/Base/Make.package b/Src/Base/Make.package
index f266cc8a306..c93bd58352c 100644
--- a/Src/Base/Make.package
+++ b/Src/Base/Make.package
@@ -17,6 +17,7 @@ C$(AMREX_BASE)_headers += AMReX_Demangle.H AMReX_Extension.H
 C$(AMREX_BASE)_headers += AMReX_GpuComplex.H
 
 C$(AMREX_BASE)_headers += AMReX_PODVector.H
+C$(AMREX_BASE)_sources += AMReX_PODVector.cpp
 
 C$(AMREX_BASE)_headers += AMReX_BlockMutex.H
 C$(AMREX_BASE)_sources += AMReX_BlockMutex.cpp