Parallelize common operations on shared memory array

src/mesh/mesh.f90

@@ -882,7 +882,7 @@ SUBROUTINE InitElemVolumes()
 #if USE_MPI
 #ifdef PARTICLES
 ! Compute-node mesh volume
-CNVolume = SUM(ElemVolume_Shared(:))
+CALL MPI_ALLREDUCE(LocalVolume,CNVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_SHARED,IERROR)
 IF (myComputeNodeRank.EQ.0) THEN
   ! All-reduce between node leaders
   CALL MPI_ALLREDUCE(CNVolume,MeshVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_LEADERS_SHARED,IERROR)

src/particles/dsmc/dsmc_symmetry.f90

@@ -110,6 +110,7 @@ SUBROUTINE DSMC_2D_InitVolumes()
 REAL                            :: radius, triarea(2)
 #if USE_MPI
 REAL                            :: CNVolume
+INTEGER                         :: offsetElemCNProc
 #endif /*USE_MPI*/
 LOGICAL                         :: SymmetryBCExists
 INTEGER                         :: firstElem, lastElem, firstSide, lastSide
@@ -223,7 +224,9 @@ SUBROUTINE DSMC_2D_InitVolumes()
 CALL BARRIER_AND_SYNC(ElemVolume_Shared_Win    ,MPI_COMM_SHARED)
 CALL BARRIER_AND_SYNC(ElemCharLength_Shared_Win,MPI_COMM_SHARED)
 ! Compute-node mesh volume
-CNVolume = SUM(ElemVolume_Shared(:))
+offsetElemCNProc = offsetElem - offsetComputeNodeElem
+CNVolume = SUM(ElemVolume_Shared(offsetElemCNProc+1:offsetElemCNProc+nElems))
+CALL MPI_ALLREDUCE(MPI_IN_PLACE,CNVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_SHARED,iError)
 IF (myComputeNodeRank.EQ.0) THEN
   ! All-reduce between node leaders
   CALL MPI_ALLREDUCE(CNVolume,MeshVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_LEADERS_SHARED,IERROR)
@@ -630,7 +633,7 @@ SUBROUTINE DSMC_2D_SetInClones()
     PartStateIntEn(1:2,PositionNbr) = ClonedParticles(iPart,DelayCounter)%PartStateIntEn(1:2)
     IF(DSMC%ElectronicModel.GT.0) THEN
       PartStateIntEn(3,PositionNbr) = ClonedParticles(iPart,DelayCounter)%PartStateIntEn(3)
-      IF ((DSMC%ElectronicModel.EQ.2).AND.(.NOT.((SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%InterID.EQ.4) & 
+      IF ((DSMC%ElectronicModel.EQ.2).AND.(.NOT.((SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%InterID.EQ.4) &
           .OR.SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%FullyIonized))) THEN
         IF(ALLOCATED(ElectronicDistriPart(PositionNbr)%DistriFunc)) DEALLOCATE(ElectronicDistriPart(PositionNbr)%DistriFunc)
         ALLOCATE(ElectronicDistriPart(PositionNbr)%DistriFunc(1:SpecDSMC(ClonedParticles(iPart,DelayCounter)%Species)%MaxElecQuant))
@@ -921,7 +924,7 @@ SUBROUTINE DSMC_2D_TreatIdenticalParticles(iPair, nPair, nPart, iElem, iPartIndx
                              + (PartState(5,iPart_p1) - PartState(5,iPart_p2))**2 &
                              + (PartState(6,iPart_p1) - PartState(6,iPart_p2))**2
   ELSE IF (iPair.LT.nPair) THEN
-    IF (.NOT.Coll_pData(iPair+1)%NeedForRec) THEN 
+    IF (.NOT.Coll_pData(iPair+1)%NeedForRec) THEN
     ! "Partner-Tausch": if there are pairs ahead in the pairing list, the next is pair is broken up and collision partners
     ! are swapped
       CollInf%Coll_CaseNum(Coll_pData(iPair)%PairType) = CollInf%Coll_CaseNum(Coll_pData(iPair)%PairType) - 1

src/particles/particle_mesh/particle_bgm.f90

@@ -1041,11 +1041,14 @@ SUBROUTINE BuildBGMAndIdentifyHaloRegion()
   nComputeNodeTotalElems = 0
   nComputeNodeTotalSides = 0
   nComputeNodeTotalNodes = 0
-  DO iElem = 1, nGlobalElems
+
+  ! Sum up all elements on the compute node
+  DO iElem = firstElem, lastElem
     IF (ElemInfo_Shared(ELEM_HALOFLAG,iElem).NE.0) THEN
       nComputeNodeTotalElems = nComputeNodeTotalElems + 1
     END IF
   END DO
+  CALL MPI_ALLREDUCE(MPI_IN_PLACE,nComputeNodeTotalElems,1,MPI_INTEGER,MPI_SUM,MPI_COMM_SHARED,iError)
 
   CALL Allocate_Shared((/nGlobalElems          /),GlobalElem2CNTotalElem_Shared_Win,GlobalElem2CNTotalElem_Shared)
   CALL MPI_WIN_LOCK_ALL(0,GlobalElem2CNTotalElem_Shared_Win,iERROR)