Skip to content

Commit

Permalink
compressZlib: don't use a SharedBuffer but a raw u8 * pointer
Browse files Browse the repository at this point in the history
Remove usage of the SharedBuffer in zlib compression which has two problems:
* We copied the whole memory block to compress it (not good with mapblocks)
* We copied sometimes strings to SharedBuffer to SharedBuffer (2nd time)

Use this method in MapNode::serializeBulk + optimize serialization but merging 3 identical loops in a single loop
  • Loading branch information
nerzhul committed Jul 27, 2017
1 parent 61e4877 commit c27504a
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 40 deletions.
35 changes: 15 additions & 20 deletions src/mapnode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -658,46 +658,41 @@ void MapNode::serializeBulk(std::ostream &os, int version,
const MapNode *nodes, u32 nodecount,
u8 content_width, u8 params_width, bool compressed)
{
if(!ser_ver_supported(version))
if (!ser_ver_supported(version))
throw VersionMismatchException("ERROR: MapNode format not supported");

sanity_check(content_width == 2);
sanity_check(params_width == 2);

// Can't do this anymore; we have 16-bit dynamically allocated node IDs
// in memory; conversion just won't work in this direction.
if(version < 24)
if (version < 24)
throw SerializationError("MapNode::serializeBulk: serialization to "
"version < 24 not possible");

SharedBuffer<u8> databuf(nodecount * (content_width + params_width));

// Serialize content
for(u32 i=0; i<nodecount; i++)
writeU16(&databuf[i*2], nodes[i].param0);
size_t databuf_size = nodecount * (content_width + params_width);
u8 *databuf = new u8[databuf_size];

// Serialize param1
u32 start1 = content_width * nodecount;
for(u32 i=0; i<nodecount; i++)
writeU8(&databuf[start1 + i], nodes[i].param1);

// Serialize param2
u32 start2 = (content_width + 1) * nodecount;
for(u32 i=0; i<nodecount; i++)

// Serialize content
for (u32 i = 0; i < nodecount; i++) {
writeU16(&databuf[i * 2], nodes[i].param0);
writeU8(&databuf[start1 + i], nodes[i].param1);
writeU8(&databuf[start2 + i], nodes[i].param2);
}

/*
Compress data to output stream
*/

if(compressed)
{
compressZlib(databuf, os);
}
if (compressed)
compressZlib(databuf, databuf_size, os);
else
{
os.write((const char*) &databuf[0], databuf.getSize());
}
os.write((const char*) &databuf[0], databuf_size);

delete [] databuf;
}

// Deserialize bulk node data
Expand Down
35 changes: 17 additions & 18 deletions src/serialization.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,

/* report a zlib or i/o error */
void zerr(int ret)
{
{
dstream<<"zerr: ";
switch (ret) {
case Z_ERRNO:
Expand All @@ -53,7 +53,7 @@ void zerr(int ret)
}
}

void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level)
{
z_stream z;
const s32 bufsize = 16384;
Expand All @@ -68,16 +68,16 @@ void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
ret = deflateInit(&z, level);
if(ret != Z_OK)
throw SerializationError("compressZlib: deflateInit failed");

// Point zlib to our input buffer
z.next_in = (Bytef*)&data[0];
z.avail_in = data.getSize();
z.avail_in = data_size;
// And get all output
for(;;)
{
z.next_out = (Bytef*)output_buffer;
z.avail_out = bufsize;

status = deflate(&z, Z_FINISH);
if(status == Z_NEED_DICT || status == Z_DATA_ERROR
|| status == Z_MEM_ERROR)
Expand All @@ -98,8 +98,7 @@ void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)

void compressZlib(const std::string &data, std::ostream &os, int level)
{
SharedBuffer<u8> databuf((u8*)data.c_str(), data.size());
compressZlib(databuf, os, level);
compressZlib((u8*)data.c_str(), data.size(), os, level);
}

void decompressZlib(std::istream &is, std::ostream &os)
Expand All @@ -120,9 +119,9 @@ void decompressZlib(std::istream &is, std::ostream &os)
ret = inflateInit(&z);
if(ret != Z_OK)
throw SerializationError("dcompressZlib: inflateInit failed");

z.avail_in = 0;

//dstream<<"initial fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;

for(;;)
Expand All @@ -143,7 +142,7 @@ void decompressZlib(std::istream &is, std::ostream &os)
//dstream<<"z.avail_in == 0"<<std::endl;
break;
}

//dstream<<"1 z.avail_in="<<z.avail_in<<std::endl;
status = inflate(&z, Z_NO_FLUSH);
//dstream<<"2 z.avail_in="<<z.avail_in<<std::endl;
Expand All @@ -163,7 +162,7 @@ void decompressZlib(std::istream &is, std::ostream &os)
if(status == Z_STREAM_END)
{
//dstream<<"Z_STREAM_END"<<std::endl;

//dstream<<"z.avail_in="<<z.avail_in<<std::endl;
//dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
// Unget all the data that inflate didn't take
Expand All @@ -178,31 +177,31 @@ void decompressZlib(std::istream &is, std::ostream &os)
throw SerializationError("decompressZlib: unget failed");
}
}

break;
}
}

inflateEnd(&z);
}

void compress(SharedBuffer<u8> data, std::ostream &os, u8 version)
void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version)
{
if(version >= 11)
{
compressZlib(data, os);
compressZlib(*data ,data.getSize(), os);
return;
}

if(data.getSize() == 0)
return;

// Write length (u32)

u8 tmp[4];
writeU32(tmp, data.getSize());
os.write((char*)tmp, 4);

// We will be writing 8-bit pairs of more_count and byte
u8 more_count = 0;
u8 current_byte = data[0];
Expand Down Expand Up @@ -242,7 +241,7 @@ void decompress(std::istream &is, std::ostream &os, u8 version)
u8 tmp[4];
is.read((char*)tmp, 4);
u32 len = readU32(tmp);

// We will be reading 8-bit pairs of more_count and byte
u32 count = 0;
for(;;)
Expand All @@ -251,7 +250,7 @@ void decompress(std::istream &is, std::ostream &os, u8 version)
u8 byte=0;

is.read((char*)&more_count, 1);

is.read((char*)&byte, 1);

if(is.eof())
Expand Down
4 changes: 2 additions & 2 deletions src/serialization.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,12 @@ inline bool ser_ver_supported(s32 v) {
Misc. serialization functions
*/

void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level = -1);
void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level = -1);
void compressZlib(const std::string &data, std::ostream &os, int level = -1);
void decompressZlib(std::istream &is, std::ostream &os);

// These choose between zlib and a self-made one according to version
void compress(SharedBuffer<u8> data, std::ostream &os, u8 version);
void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version);
//void compress(const std::string &data, std::ostream &os, u8 version);
void decompress(std::istream &is, std::ostream &os, u8 version);

Expand Down

0 comments on commit c27504a

Please sign in to comment.