Skip to content

Commit

Permalink
fixing work
Browse files Browse the repository at this point in the history
  • Loading branch information
pengfej committed May 7, 2022
1 parent e318af3 commit d077880
Showing 1 changed file with 42 additions and 6 deletions.
48 changes: 42 additions & 6 deletions source/base/data_out_base.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7689,6 +7689,10 @@ DataOutInterface<dim, spacedim>::write_vtu_in_parallel(
#else

const int myrank = Utilities::MPI::this_mpi_process(comm);
int rank;
MPI_Comm_size(comm, &rank);
MPI_Offset offset;
std::uint64_t size_placeholder;

MPI_Info info;
int ierr = MPI_Info_create(&info);
Expand Down Expand Up @@ -7718,7 +7722,7 @@ DataOutInterface<dim, spacedim>::write_vtu_in_parallel(
// Write the header on rank 0 and automatically move the
// shared file pointer to the location after header;
ierr = MPI_File_write_at(
fh, ss.str().c_str(), header_size, MPI_CHAR, MPI_STATUS_IGNORE);
fh, 0, ss.str().c_str(), header_size, MPI_CHAR, MPI_STATUS_IGNORE);
AssertThrowMPI(ierr);
}

Expand All @@ -7733,6 +7737,9 @@ DataOutInterface<dim, spacedim>::write_vtu_in_parallel(
// the first piece written. But if nobody has any pieces to write (file is
// empty), let processor 0 write their empty data, otherwise the vtk file is
// invalid.



std::stringstream ss;
if (my_n_patches > 0 || (global_n_patches == 0 && myrank == 0))
DataOutBase::write_vtu_main(patches,
Expand All @@ -7741,19 +7748,48 @@ DataOutInterface<dim, spacedim>::write_vtu_in_parallel(
vtk_flags,
ss);

ierr = MPI_File_write_at_all_c(
fh, ss.str().c_str(), ss.str().size(), MPI_CHAR, MPI_STATUS_IGNORE);
//using prefix sum to find specific offset to write at.

const std::uint64_t size_on_proc = ss.str().size();
size_placeholder = size_on_proc;
std::uint64_t prefix_sum = 0;
ierr = MPI_Exscan(&size_on_proc,
&prefix_sum,
1,
MPI_CHAR,
MPI_SUM,
comm);
AssertThrowMPI(ierr);

ierr = MPI_Bcast(&header_size, 1, MPI_UNSIGNED, 0, comm);
AssertThrowMPI(ierr);

offset = static_cast<MPI_Offset>(header_size) * sizeof( char ) + prefix_sum;

ierr = MPI_File_write_at_all(
fh, offset, ss.str().c_str(), ss.str().size(), MPI_CHAR, MPI_STATUS_IGNORE);
AssertThrowMPI(ierr);
}

// write footer
if (myrank == 0)
if (myrank == rank-1)
{
std::stringstream ss;
DataOutBase::write_vtu_footer(ss);
unsigned int footer_size = ss.str().size();
ierr = MPI_File_write_at(
fh, ss.str().c_str(), footer_size, MPI_CHAR, MPI_STATUS_IGNORE);

ierr = MPI_Bcast(&offset, 1, MPI_UNSIGNED_LONG_LONG, rank-1,comm);
AssertThrowMPI(ierr);

// std::uint64_t size_on_proc;
// ierr = MPI_Bcast(&size_on_proc, 1, MPI_INT64_T, rank-1, comm);
ierr = MPI_Bcast(&size_placeholder, 1, MPI_INT64_T, rank-1, comm);

offset = offset + static_cast<MPI_Offset>(size_placeholder) * sizeof( char );
// offset += size_on_proc;

// Writing Footer.
ierr = MPI_File_write_at(fh, offset, ss.str().c_str(), footer_size, MPI_CHAR, MPI_STATUS_IGNORE);
AssertThrowMPI(ierr);
}
ierr = MPI_File_close(&fh);
Expand Down

0 comments on commit d077880

Please sign in to comment.