Merge pull request #4882 from psychocoderHPC/topic-optimizeOpenPMDIO

optimize OpenPMD IO
ComputationalRadiationPhysics · Apr 29, 2024 · d24fe3f · d24fe3f
2 parents c52f902 + 911fbb6
commit d24fe3f
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 23 deletions.
diff --git a/include/picongpu/plugins/openPMD/WriteSpecies.hpp b/include/picongpu/plugins/openPMD/WriteSpecies.hpp
@@ -480,6 +480,8 @@ namespace picongpu
                             globalNumParticles,
                             myParticleOffset + particleOffset);
 
+                        params->openPMDSeries->flush(PreferredFlushTarget::Disk);
+
                         log<picLog::INPUT_OUTPUT>("openPMD: (end) write particle records for %1%, dumping round %2%")
                             % T_SpeciesFilter::getName() % dumpIteration;
 

diff --git a/include/picongpu/plugins/openPMD/writer/ParticleAttribute.hpp b/include/picongpu/plugins/openPMD/writer/ParticleAttribute.hpp
@@ -48,6 +48,8 @@ namespace picongpu
         struct ParticleAttribute
         {
             /** write attribute to openPMD series
+             *
+             * @attention This is a MPI_Collective operation and all MPI ranks must participate.
              *
              * @param params wrapped params
              * @param elements elements of this attribute
@@ -98,8 +100,6 @@ namespace picongpu
 
                 log<picLog::INPUT_OUTPUT>("openPMD:  (begin) write species attribute: %1%") % Identifier::getName();
 
-                std::shared_ptr<ComponentType> storeBfr;
-
                 for(uint32_t d = 0; d < components; d++)
                 {
                     ::openPMD::RecordComponent recordComponent
@@ -115,39 +115,23 @@ namespace picongpu
 
                     if(elements == 0)
                     {
-                        params->openPMDSeries->flush(PreferredFlushTarget::Disk);
                         continue;
                     }
 
                     ValueType* dataPtr = frame.getIdentifier(Identifier()).getPointer(); // can be moved up?
                     // ask openPMD to create a buffer for us
-                    // in some backends (ADIOS2), this allows avoiding memcopies
-                    auto span = recordComponent
-                                    .storeChunk<ComponentType>(
-                                        ::openPMD::Offset{globalOffset},
-                                        ::openPMD::Extent{elements},
-                                        [&storeBfr](size_t size)
-                                        {
-                                            // if there is no special backend support for creating buffers,
-                                            // reuse the storeBfr
-                                            if(!storeBfr && size > 0)
-                                            {
-                                                storeBfr = std::shared_ptr<ComponentType>{
-                                                    new ComponentType[size],
-                                                    [](ComponentType* ptr) { delete[] ptr; }};
-                                            }
-                                            return storeBfr;
-                                        })
-                                    .currentBuffer();
+                    // in some backends (ADIOS2), this allows avoiding memory copies
+                    auto span
+                        = recordComponent
+                              .storeChunk<ComponentType>(::openPMD::Offset{globalOffset}, ::openPMD::Extent{elements})
+                              .currentBuffer();
 
 /* copy strided data from source to temporary buffer */
 #pragma omp parallel for simd
                     for(size_t i = 0; i < elements; ++i)
                     {
                         span[i] = reinterpret_cast<ComponentType*>(dataPtr)[d + i * components];
                     }
-
-                    params->openPMDSeries->flush(PreferredFlushTarget::Disk);
                 }
 
                 log<picLog::INPUT_OUTPUT>("openPMD:  ( end ) write species attribute: %1%") % Identifier::getName();