Add dUSE_LOG_FINEGRAIN for logging of fine-grained operations.

This is disabled by default because it has significant overhead for
dEFSApply and dFSMatSetValuesBlockedExpanded.

This patch also updates dFSMatSetValuesBlockedExpanded to specialize on
MATAIJ for MatGetRowIJ because profiling is done otherwise.

CMakeLists.txt

@@ -13,9 +13,10 @@ endif ()
 
 #find_package (Doxygen)
 
-option (dUSE_DEBUG         "Compile Dohp with extra debugging" ON)
-option (dUSE_PARALLEL_HDF5 "Write HDF5 files in parallel"      OFF)
-option (Dohp_BUILD_TESTS   "Build tests"                       ON)
+option (dUSE_DEBUG              "Compile Dohp with extra debugging"                              ON)
+option (dUSE_LOG_FINEGRAIN      "Log fine grained operations for profiling (slower)"             OFF)
+option (dUSE_PARALLEL_HDF5      "Write HDF5 files in parallel"                                   OFF)
+option (Dohp_BUILD_TESTS        "Build tests"                                                    ON)
 if (Dohp_BUILD_TESTS)
   option (Dohp_TESTS_MALLOC_DUMP "Run all tests with -malloc_dump" ON)
   mark_as_advanced (Dohp_TESTS_MALLOC_DUMP)

dohpconfig.h.in

@@ -3,6 +3,7 @@
 
 #cmakedefine dUSE_DEBUG
 #cmakedefine dUSE_PARALLEL_HDF5
+#cmakedefine dUSE_LOG_FINEGRAIN
 #cmakedefine dUSE_VALGRIND
 
 #endif

src/fs/interface/fs.c

@@ -700,6 +700,12 @@ dErr dFSGetMatrix(dFS fs,const MatType mtype,Mat *inJ)
 extern PetscErrorCode MatGetArray_SeqAIJ(Mat A,PetscScalar *array[]);
 extern PetscErrorCode MatRestoreArray_SeqAIJ(Mat A,PetscScalar *array[]);
 
+// We call these directly because otherwise profiling events are done, which adds overhead.
+#if !defined dUSE_LOG_FINEGRAIN
+extern PetscErrorCode MatGetRowIJ_SeqAIJ(Mat A,PetscInt oshift,PetscBool symmetric,PetscBool inodecompressed,PetscInt *m,PetscInt *ia[],PetscInt *ja[],PetscBool *done);
+extern PetscErrorCode MatRestoreRowIJ_SeqAIJ(Mat A,PetscInt oshift,PetscBool symmetric,PetscBool inodecompressed,PetscInt *n,PetscInt *ia[],PetscInt *ja[],PetscBool *done);
+#endif
+
 dErr dFSMatSetValuesBlockedExpanded(dFS fs,Mat A,dInt m,const dInt idxm[],dInt n,const dInt idxn[],const dScalar v[],InsertMode imode)
 {
   dInt lidxms[128],lidxns[128];
@@ -718,10 +724,16 @@ dErr dFSMatSetValuesBlockedExpanded(dFS fs,Mat A,dInt m,const dInt idxm[],dInt n
   dValidPointer(idxn,6);
   dValidPointer(v,7);
   err = dFSGetBlockSize(fs,&bs);dCHK(err);
+#if defined dUSE_LOG_FINEGRAIN
   err = PetscLogEventBegin(dLOG_FSMatSetValuesExpanded,fs,A,0,0);dCHK(err);
+#endif
   err = MatMAIJGetAIJ(fs->assemblefull?fs->E:fs->Ep,&E);dCHK(err); /* Does not reference so do not destroy or return E */
+#if defined dUSE_DEBUG
   err = MatGetRowIJ(E,0,dFALSE,dFALSE,&cn,&ci,&cj,&done);dCHK(err);
   if (!done) dERROR(PETSC_COMM_SELF,1,"Could not get indices");
+#else
+  err = MatGetRowIJ_SeqAIJ(E,0,dFALSE,dFALSE,&cn,&ci,&cj,&done);dCHK(err);
+#endif
   err = MatGetArray_SeqAIJ(E,&ca);dCHK(err);
   for (i=0,lm=0; i<m; i++) {
     /* Count the number of columns in constraint matrix for each row of input matrix, this will be the total number of
@@ -766,8 +778,12 @@ dErr dFSMatSetValuesBlockedExpanded(dFS fs,Mat A,dInt m,const dInt idxm[],dInt n
   }
 
   err = MatRestoreArray_SeqAIJ(E,&ca);dCHK(err);
+#if defined dUSE_DEBUG
   err = MatRestoreRowIJ(E,0,dFALSE,dFALSE,&cn,&ci,&cj,&done);dCHK(err);
   if (!done) dERROR(PETSC_COMM_SELF,1,"Failed to return indices");
+#else
+  err = MatRestoreRowIJ_SeqAIJ(E,0,dFALSE,dFALSE,&cn,&ci,&cj,&done);dCHK(err);
+#endif
   if (fs->assemblereduced) {
     dInt brow[lm],bcol[ln];
     dScalar bval[lm*ln];
@@ -789,6 +805,8 @@ dErr dFSMatSetValuesBlockedExpanded(dFS fs,Mat A,dInt m,const dInt idxm[],dInt n
   if (lidxn != lidxns) {err = dFree(lidxn);dCHK(err);}
   if (lv != lvs)       {err = dFree(lv);dCHK(err);}
   if (lvt != lvts)     {err = dFree(lvt);dCHK(err);}
+#if defined dUSE_LOG_FINEGRAIN
   err = PetscLogEventEnd(dLOG_FSMatSetValuesExpanded,fs,A,0,0);dCHK(err);
+#endif
   dFunctionReturn(0);
 }

src/jacobi/interface/jacobi.c

@@ -512,9 +512,13 @@ dErr dEFSApply(dEFS efs,const dReal mapdata[],dInt dofs,const dScalar in[],dScal
   if (dofs < 1) dERROR(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"dofs %D, must be positive",dofs);
   dValidScalarPointer(in,4);
   dValidScalarPointer(out,5);
+#if defined dUSE_LOG_FINEGRAIN
   err = PetscLogEventBegin(dLOG_EFSApply,0,0,0,0);dCHK(err);
+#endif
   err = (*efs->ops.apply)(efs,mapdata,dofs,in,out,amode,imode);dCHK(err);
+#if defined dUSE_LOG_FINEGRAIN
   err = PetscLogEventEnd(dLOG_EFSApply,0,0,0,0);dCHK(err);
+#endif
   dFunctionReturn(0);
 }