Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add DataOutBase::CompressionLevel::ascii #14284

Merged
merged 1 commit into from
Sep 20, 2022
Merged

Add DataOutBase::CompressionLevel::ascii #14284

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 9 additions & 2 deletions include/deal.II/base/data_out_base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,10 @@ namespace DataOutBase
{
/**
* An enum for different levels of compression used in several places
* to determine zlib compression levels.
* to determine zlib compression levels for binary output. At some
* places, it is possible to output the data also as plain text as
* an alternative, which is convenient for debugging. We use
* this flag to indicate such an output as well.
*/
enum class CompressionLevel
{
Expand All @@ -243,7 +246,11 @@ namespace DataOutBase
* Use the default compression algorithm. This is a compromise between
* speed and file size.
*/
default_compression
default_compression,
/**
* Output as plain text (ASCII) if available.
*/
plain_text
};


Expand Down
204 changes: 128 additions & 76 deletions source/base/data_out_base.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1921,31 +1921,39 @@ namespace
void
VtuStream::write_point(const unsigned int, const Point<dim> &p)
{
#if !defined(DEAL_II_WITH_ZLIB)
// write out coordinates
stream << p;
// fill with zeroes
for (unsigned int i = dim; i < 3; ++i)
stream << " 0";
stream << '\n';
#else
// if we want to compress, then first collect all the data in an array
for (unsigned int i = 0; i < dim; ++i)
vertices.push_back(p[i]);
for (unsigned int i = dim; i < 3; ++i)
vertices.push_back(0);
#ifdef DEAL_II_WITH_ZLIB
if (flags.compression_level != DataOutBase::CompressionLevel::plain_text)
{
// if we want to compress, then first collect all the data in an array
for (unsigned int i = 0; i < dim; ++i)
vertices.push_back(p[i]);
for (unsigned int i = dim; i < 3; ++i)
vertices.push_back(0);
}
else
#endif
{
// write out coordinates
stream << p;
// fill with zeroes
for (unsigned int i = dim; i < 3; ++i)
stream << " 0";
stream << '\n';
}
}


void
VtuStream::flush_points()
{
#ifdef DEAL_II_WITH_ZLIB
// compress the data we have in memory and write them to the stream. then
// release the data
*this << vertices << '\n';
vertices.clear();
if (flags.compression_level != DataOutBase::CompressionLevel::plain_text)
{
// compress the data we have in memory and write them to the stream.
// then release the data
*this << vertices << '\n';
vertices.clear();
}
#endif
}

Expand All @@ -1958,41 +1966,47 @@ namespace
unsigned int d2,
unsigned int d3)
{
#if !defined(DEAL_II_WITH_ZLIB)
stream << start;
if (dim >= 1)
#ifdef DEAL_II_WITH_ZLIB
if (flags.compression_level != DataOutBase::CompressionLevel::plain_text)
{
stream << '\t' << start + d1;
if (dim >= 2)
cells.push_back(start);
if (dim >= 1)
{
stream << '\t' << start + d2 + d1 << '\t' << start + d2;
if (dim >= 3)
cells.push_back(start + d1);
if (dim >= 2)
{
stream << '\t' << start + d3 << '\t' << start + d3 + d1 << '\t'
<< start + d3 + d2 + d1 << '\t' << start + d3 + d2;
cells.push_back(start + d2 + d1);
cells.push_back(start + d2);
if (dim >= 3)
{
cells.push_back(start + d3);
cells.push_back(start + d3 + d1);
cells.push_back(start + d3 + d2 + d1);
cells.push_back(start + d3 + d2);
}
}
}
}
stream << '\n';
#else
cells.push_back(start);
if (dim >= 1)
else
#endif
{
cells.push_back(start + d1);
if (dim >= 2)
stream << start;
if (dim >= 1)
{
cells.push_back(start + d2 + d1);
cells.push_back(start + d2);
if (dim >= 3)
stream << '\t' << start + d1;
if (dim >= 2)
{
cells.push_back(start + d3);
cells.push_back(start + d3 + d1);
cells.push_back(start + d3 + d2 + d1);
cells.push_back(start + d3 + d2);
stream << '\t' << start + d2 + d1 << '\t' << start + d2;
if (dim >= 3)
{
stream << '\t' << start + d3 << '\t' << start + d3 + d1
<< '\t' << start + d3 + d2 + d1 << '\t'
<< start + d3 + d2;
}
}
}
stream << '\n';
}
#endif
}

void
Expand All @@ -2005,17 +2019,23 @@ namespace

static const std::array<unsigned int, 5> table = {{0, 1, 3, 2, 4}};

#if !defined(DEAL_II_WITH_ZLIB)
for (unsigned int i = 0; i < n_points; ++i)
stream << '\t'
<< start +
(reference_cell == ReferenceCells::Pyramid ? table[i] : i);
stream << '\n';
#else
for (unsigned int i = 0; i < n_points; ++i)
cells.push_back(
start + (reference_cell == ReferenceCells::Pyramid ? table[i] : i));
#ifdef DEAL_II_WITH_ZLIB
if (flags.compression_level != DataOutBase::CompressionLevel::plain_text)
{
for (unsigned int i = 0; i < n_points; ++i)
cells.push_back(
start + (reference_cell == ReferenceCells::Pyramid ? table[i] : i));
}
else
#endif
{
for (unsigned int i = 0; i < n_points; ++i)
stream << '\t'
<< start + (reference_cell == ReferenceCells::Pyramid ?
table[i] :
i);
stream << '\n';
}
}

template <int dim>
Expand All @@ -2024,24 +2044,32 @@ namespace
const unsigned int start,
const std::vector<unsigned> &connectivity)
{
#if !defined(DEAL_II_WITH_ZLIB)
for (const auto &c : connectivity)
stream << '\t' << start + c;
stream << '\n';
#else
for (const auto &c : connectivity)
cells.push_back(start + c);
#ifdef DEAL_II_WITH_ZLIB
if (flags.compression_level != DataOutBase::CompressionLevel::plain_text)
{
for (const auto &c : connectivity)
cells.push_back(start + c);
}
else
#endif
{
for (const auto &c : connectivity)
stream << '\t' << start + c;
stream << '\n';
}
}

void
VtuStream::flush_cells()
{
#ifdef DEAL_II_WITH_ZLIB
// compress the data we have in memory and write them to the stream. then
// release the data
*this << cells << '\n';
cells.clear();
if (flags.compression_level != DataOutBase::CompressionLevel::plain_text)
{
// compress the data we have in memory and write them to the stream.
// then release the data
*this << cells << '\n';
cells.clear();
}
#endif
}

Expand All @@ -2051,13 +2079,18 @@ namespace
VtuStream::operator<<(const std::vector<T> &data)
{
#ifdef DEAL_II_WITH_ZLIB
// compress the data we have in memory and write them to the stream. then
// release the data
write_compressed_block(data, flags, stream);
#else
for (unsigned int i = 0; i < data.size(); ++i)
stream << data[i] << ' ';
if (flags.compression_level != DataOutBase::CompressionLevel::plain_text)
{
// compress the data we have in memory and write them to the stream.
// then release the data
write_compressed_block(data, flags, stream);
}
else
#endif
{
for (unsigned int i = 0; i < data.size(); ++i)
stream << data[i] << ' ';
}

return stream;
}
Expand Down Expand Up @@ -5804,7 +5837,8 @@ namespace DataOutBase
out << "\n-->\n";
out << "<VTKFile type=\"UnstructuredGrid\" version=\"0.1\"";
#ifdef DEAL_II_WITH_ZLIB
out << " compressor=\"vtkZLibDataCompressor\"";
if (flags.compression_level != CompressionLevel::plain_text)
out << " compressor=\"vtkZLibDataCompressor\"";
#endif
#ifdef DEAL_II_WORDS_BIGENDIAN
out << " byte_order=\"BigEndian\"";
Expand Down Expand Up @@ -5988,11 +6022,11 @@ namespace DataOutBase
AssertDimension(n_data_sets, patches[0].data.n_rows())
}

const char *ascii_or_binary =
#ifdef DEAL_II_WITH_ZLIB
const char *ascii_or_binary = "binary";
#else
const char * ascii_or_binary = "ascii";
(flags.compression_level != CompressionLevel::plain_text) ? "binary" :
#endif
"ascii";


// first count the number of cells and cells for later use
Expand Down Expand Up @@ -6057,11 +6091,7 @@ namespace DataOutBase

// std::uint8_t might be an alias to unsigned char which is then not printed
// as ascii integers
#ifdef DEAL_II_WITH_ZLIB
std::vector<std::uint8_t> cell_types;
#else
std::vector<unsigned int> cell_types;
#endif
cell_types.reserve(n_cells);

unsigned int first_vertex_of_patch = 0;
Expand Down Expand Up @@ -6089,7 +6119,21 @@ namespace DataOutBase
<< ascii_or_binary << "\">\n";

// this should compress well :-)
vtu_out << cell_types;
#ifdef DEAL_II_WITH_ZLIB
if (flags.compression_level != CompressionLevel::plain_text)
{
std::vector<uint8_t> cell_types_uint8_t(cell_types.size());
for (unsigned int i = 0; i < cell_types.size(); ++i)
cell_types_uint8_t[i] = static_cast<std::uint8_t>(cell_types[i]);

vtu_out << cell_types_uint8_t;
}
else
#endif
{
vtu_out << cell_types;
}

out << '\n';
out << " </DataArray>\n";
out << " </Cells>\n";
Expand Down Expand Up @@ -7642,6 +7686,9 @@ namespace DataOutBase
{
boost::iostreams::filtering_ostream f;

AssertThrow(compression != CompressionLevel::plain_text,
ExcNotImplemented());

if (compression != CompressionLevel::no_compression)
# ifdef DEAL_II_WITH_ZLIB
f.push(boost::iostreams::zlib_compressor(
Expand Down Expand Up @@ -9335,6 +9382,11 @@ DataOutReader<dim, spacedim>::read_whole_parallel_file(std::istream &in)
std::vector<char> temp_buffer(chunk_sizes[n]);
in.read(temp_buffer.data(), chunk_sizes[n]);

AssertThrow(static_cast<DataOutBase::CompressionLevel>(
header.compression) !=
DataOutBase::CompressionLevel::plain_text,
ExcNotImplemented());

boost::iostreams::filtering_istreambuf f;
if (static_cast<DataOutBase::CompressionLevel>(header.compression) !=
DataOutBase::CompressionLevel::no_compression)
Expand Down
5 changes: 4 additions & 1 deletion tests/data_out/data_out_base_vtu_compression_levels.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ template <int dim, int spacedim>
void
check_all(std::ostream &log)
{
for (unsigned int i = 0; i < 4; ++i)
for (unsigned int i = 0; i < 5; ++i)
{
DataOutBase::VtkFlags flags;
switch (i)
Expand All @@ -76,6 +76,9 @@ check_all(std::ostream &log)
flags.compression_level =
DataOutBase::CompressionLevel::default_compression;
break;
case (4):
flags.compression_level = DataOutBase::CompressionLevel::plain_text;
break;
default:
Assert(false, ExcInternalError());
}
Expand Down