Skip to content
Permalink
Browse files
Add support for map block version 29
  • Loading branch information
sfan5 committed Sep 1, 2021
1 parent 5c435f6 commit b0ca3d706604d6c1878f03990fa36345872df664
Showing with 163 additions and 37 deletions.
  1. +44 −32 BlockDecoder.cpp
  2. +8 −1 CMakeLists.txt
  3. +58 −0 ZstdDecompressor.cpp
  4. +24 −0 cmake/FindZstd.cmake
  5. +6 −3 include/BlockDecoder.h
  6. +22 −0 include/ZstdDecompressor.h
  7. +1 −1 util/ci/script.sh
@@ -1,4 +1,3 @@
#include <stdint.h>
#include <string>
#include <iostream>
#include <sstream>
@@ -39,7 +38,7 @@ void BlockDecoder::reset()

m_version = 0;
m_contentWidth = 0;
m_mapData = ustring();
m_mapData.clear();
}

void BlockDecoder::decode(const ustring &datastr)
@@ -49,20 +48,53 @@ void BlockDecoder::decode(const ustring &datastr)
// TODO: bounds checks

uint8_t version = data[0];
//uint8_t flags = data[1];
if (version < 22) {
std::ostringstream oss;
oss << "Unsupported map version " << (int)version;
throw std::runtime_error(oss.str());
}
m_version = version;

ustring datastr2;
if (version >= 29) {
// decompress whole block at once
m_zstd_decompressor.setData(data, length, 1);
datastr2 = m_zstd_decompressor.decompress();
data = datastr2.c_str();
length = datastr2.size();
}

size_t dataOffset = 0;
if (version >= 27)
if (version >= 29)
dataOffset = 7;
else if (version >= 27)
dataOffset = 4;
else
dataOffset = 2;

auto decode_mapping = [&] () {
dataOffset++; // mapping version
uint16_t numMappings = readU16(data + dataOffset);
dataOffset += 2;
for (int i = 0; i < numMappings; ++i) {
uint16_t nodeId = readU16(data + dataOffset);
dataOffset += 2;
uint16_t nameLen = readU16(data + dataOffset);
dataOffset += 2;
std::string name(reinterpret_cast<const char *>(data) + dataOffset, nameLen);
if (name == "air")
m_blockAirId = nodeId;
else if (name == "ignore")
m_blockIgnoreId = nodeId;
else
m_nameMap[nodeId] = name;
dataOffset += nameLen;
}
};

if (version >= 29)
decode_mapping();

uint8_t contentWidth = data[dataOffset];
dataOffset++;
uint8_t paramsWidth = data[dataOffset];
@@ -73,14 +105,20 @@ void BlockDecoder::decode(const ustring &datastr)
throw std::runtime_error("unsupported map version (paramsWidth)");
m_contentWidth = contentWidth;

if (version >= 29) {
m_mapData.resize((contentWidth + paramsWidth) * 4096);
m_mapData.assign(data + dataOffset, m_mapData.size());
return; // we have read everything we need and can return early
}

// version < 29
ZlibDecompressor decompressor(data, length);
decompressor.setSeekPos(dataOffset);
m_mapData = decompressor.decompress();
decompressor.decompress(); // unused metadata
dataOffset = decompressor.seekPos();

// Skip unused data
// Skip unused node timers
if (version == 23)
dataOffset += 1;
if (version == 24) {
@@ -104,33 +142,7 @@ void BlockDecoder::decode(const ustring &datastr)
dataOffset += 4; // Skip timestamp

// Read mapping
{
dataOffset++; // mapping version
uint16_t numMappings = readU16(data + dataOffset);
dataOffset += 2;
for (int i = 0; i < numMappings; ++i) {
uint16_t nodeId = readU16(data + dataOffset);
dataOffset += 2;
uint16_t nameLen = readU16(data + dataOffset);
dataOffset += 2;
std::string name(reinterpret_cast<const char *>(data) + dataOffset, nameLen);
if (name == "air")
m_blockAirId = nodeId;
else if (name == "ignore")
m_blockIgnoreId = nodeId;
else
m_nameMap[nodeId] = name;
dataOffset += nameLen;
}
}

// Node timers
if (version >= 25) {
uint8_t timerLength = data[dataOffset++];
uint16_t numTimers = readU16(data + dataOffset);
dataOffset += 2;
dataOffset += numTimers * timerLength;
}
decode_mapping();
}

bool BlockDecoder::isEmpty() const
@@ -43,7 +43,7 @@ if(NOT CUSTOM_DOCDIR STREQUAL "")
message(STATUS "Using DOCDIR=${DOCDIR}")
endif()

#set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)

# Libraries: gd

@@ -59,6 +59,10 @@ endif(NOT LIBGD_LIBRARY OR NOT LIBGD_INCLUDE_DIR)

find_package(ZLIB REQUIRED)

# Libraries: zstd

find_package(Zstd REQUIRED)

# Libraries: sqlite3

find_library(SQLITE3_LIBRARY sqlite3)
@@ -148,6 +152,7 @@ include_directories(
${SQLITE3_INCLUDE_DIR}
${LIBGD_INCLUDE_DIR}
${ZLIB_INCLUDE_DIR}
${ZSTD_INCLUDE_DIR}
)

configure_file(
@@ -171,6 +176,7 @@ add_executable(minetestmapper
PlayerAttributes.cpp
TileGenerator.cpp
ZlibDecompressor.cpp
ZstdDecompressor.cpp
Image.cpp
mapper.cpp
util.cpp
@@ -188,6 +194,7 @@ target_link_libraries(
${REDIS_LIBRARY}
${LIBGD_LIBRARY}
${ZLIB_LIBRARY}
${ZSTD_LIBRARY}
)

# Installing & Packaging
@@ -0,0 +1,58 @@
#include <zstd.h>
#include "ZstdDecompressor.h"

ZstdDecompressor::ZstdDecompressor():
m_data(nullptr),
m_seekPos(0),
m_size(0)
{
m_stream = ZSTD_createDStream();
}

ZstdDecompressor::~ZstdDecompressor()
{
ZSTD_freeDStream(reinterpret_cast<ZSTD_DStream*>(m_stream));
}

void ZstdDecompressor::setData(const u8 *data, size_t size, size_t seekPos)
{
m_data = data;
m_seekPos = seekPos;
m_size = size;
}

std::size_t ZstdDecompressor::seekPos() const
{
return m_seekPos;
}

ustring ZstdDecompressor::decompress()
{
ZSTD_DStream *stream = reinterpret_cast<ZSTD_DStream*>(m_stream);
ZSTD_inBuffer inbuf = { m_data, m_size, m_seekPos };

ustring buffer;
constexpr size_t BUFSIZE = 32 * 1024;

buffer.resize(BUFSIZE);
ZSTD_outBuffer outbuf = { &buffer[0], buffer.size(), 0 };

ZSTD_initDStream(stream);

size_t ret;
do {
ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
if (outbuf.size == outbuf.pos) {
outbuf.size += BUFSIZE;
buffer.resize(outbuf.size);
outbuf.dst = &buffer[0];
}
} while (ret != 0);
if (ZSTD_isError(ret))
throw DecompressError();

m_seekPos = inbuf.pos;
buffer.resize(outbuf.pos);

return buffer;
}
@@ -0,0 +1,24 @@
mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR)

find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)

find_library(ZSTD_LIBRARY NAMES zstd)

if(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY)
# Check that the API we use exists
include(CheckSymbolExists)
unset(HAVE_ZSTD_INITDSTREAM CACHE)
set(CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
set(CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
check_symbol_exists(ZSTD_initDStream zstd.h HAVE_ZSTD_INITDSTREAM)
unset(CMAKE_REQUIRED_INCLUDES)
unset(CMAKE_REQUIRED_LIBRARIES)

if(NOT HAVE_ZSTD_INITDSTREAM)
unset(ZSTD_INCLUDE_DIR CACHE)
unset(ZSTD_LIBRARY CACHE)
endif()
endif()

include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
@@ -1,8 +1,9 @@
#pragma once

#include <cstdint>
#include <unordered_map>

#include "types.h"
#include <ZstdDecompressor.h>

class BlockDecoder {
public:
@@ -17,9 +18,11 @@ class BlockDecoder {
private:
typedef std::unordered_map<uint16_t, std::string> NameMap;
NameMap m_nameMap;
int m_blockAirId;
int m_blockIgnoreId;
uint16_t m_blockAirId, m_blockIgnoreId;

u8 m_version, m_contentWidth;
ustring m_mapData;

// one instance for performance
ZstdDecompressor m_zstd_decompressor;
};
@@ -0,0 +1,22 @@
#pragma once

#include <cstdlib>
#include <string>
#include "types.h"

class ZstdDecompressor
{
public:
class DecompressError {};

ZstdDecompressor();
~ZstdDecompressor();
void setData(const u8 *data, size_t size, size_t seekPos);
size_t seekPos() const;
ustring decompress();

private:
void *m_stream; // ZSTD_DStream
const u8 *m_data;
size_t m_seekPos, m_size;
};
@@ -1,7 +1,7 @@
#!/bin/bash -e

install_linux_deps() {
local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev)
local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev libzstd-dev)

sudo apt-get update
sudo apt-get install -y --no-install-recommends ${pkgs[@]} "$@"

0 comments on commit b0ca3d7

Please sign in to comment.