Skip to content
Permalink
Browse files
Implement AreaStore serialization
  • Loading branch information
ShadowNinja committed Mar 7, 2016
1 parent c4b7afe commit 821551a2669123ac9a476894d65b5efe10026040
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 92 deletions.
@@ -2737,6 +2737,10 @@ If you chose the parameter-less constructor, a fast implementation will be autom
block_radius = number, -- the radius (in nodes) of the areas the cache generates prefiltered lists for, minimum 16, default 64
limit = number, -- the cache's size, minimum 20, default 1000
}
* `to_string()`: Experimental. Returns area store serialized as a (binary) string.
* `to_file(filename)`: Experimental. Like `to_string()`, but writes the data to a file.
* `from_string(str)`: Experimental. Deserializes string and loads it into the AreaStore. Returns success and, optionally, an error message.
* `from_file(filename)`: Experimental. Like `from_string()`, but reads the data from a file.

### `ItemStack`
An `ItemStack` is a stack of items.
@@ -70,6 +70,22 @@ static inline void push_areas(lua_State *L, const std::vector<Area *> &areas,
}
}

// Deserializes value and handles errors
static int deserialization_helper(lua_State *L, AreaStore *as,
std::istream &is)
{
try {
as->deserialize(is);
} catch (const SerializationError &e) {
lua_pushboolean(L, false);
lua_pushstring(L, e.what());
return 2;
}

lua_pushboolean(L, true);
return 1;
}

// garbage collector
int LuaAreaStore::gc_object(lua_State *L)
{
@@ -217,17 +233,15 @@ int LuaAreaStore::l_set_cache_params(lua_State *L)
return 0;
}

#if 0
// to_string()
int LuaAreaStore::l_to_string(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;

LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;

std::ostringstream os(std::ios_base::binary);
ast->serialize(os);
o->as->serialize(os);
std::string str = os.str();

lua_pushlstring(L, str.c_str(), str.length());
@@ -258,16 +272,12 @@ int LuaAreaStore::l_from_string(lua_State *L)
NO_MAP_LOCK_REQUIRED;

LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;

size_t len;
const char *str = luaL_checklstring(L, 2, &len);

std::istringstream is(std::string(str, len), std::ios::binary);
bool success = ast->deserialize(is);

lua_pushboolean(L, success);
return 1;
return deserialization_helper(L, o->as, is);
}

// from_file(filename)
@@ -276,18 +286,13 @@ int LuaAreaStore::l_from_file(lua_State *L)
NO_MAP_LOCK_REQUIRED;

LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;

const char *filename = luaL_checkstring(L, 2);
CHECK_SECURE_PATH_OPTIONAL(L, filename);

std::ifstream is(filename, std::ios::binary);
bool success = ast->deserialize(is);

lua_pushboolean(L, success);
return 1;
return deserialization_helper(L, o->as, is);
}
#endif

LuaAreaStore::LuaAreaStore()
{
@@ -377,9 +382,9 @@ const luaL_reg LuaAreaStore::methods[] = {
luamethod(LuaAreaStore, reserve),
luamethod(LuaAreaStore, remove_area),
luamethod(LuaAreaStore, set_cache_params),
/* luamethod(LuaAreaStore, to_string),
luamethod(LuaAreaStore, to_string),
luamethod(LuaAreaStore, to_file),
luamethod(LuaAreaStore, from_string),
luamethod(LuaAreaStore, from_file),*/
luamethod(LuaAreaStore, from_file),
{0,0}
};
@@ -43,11 +43,11 @@ class LuaAreaStore : public ModApiBase {

static int l_set_cache_params(lua_State *L);

/* static int l_to_string(lua_State *L);
static int l_to_string(lua_State *L);
static int l_to_file(lua_State *L);

static int l_from_string(lua_State *L);
static int l_from_file(lua_State *L); */
static int l_from_file(lua_State *L);

public:
AreaStore *as;
@@ -31,6 +31,7 @@ class TestAreaStore : public TestBase {
void genericStoreTest(AreaStore *store);
void testVectorStore();
void testSpatialStore();
void testSerialization();
};

static TestAreaStore g_test_instance;
@@ -41,6 +42,7 @@ void TestAreaStore::runTests(IGameDef *gamedef)
#if USE_SPATIAL
TEST(testSpatialStore);
#endif
TEST(testSerialization);
}

////////////////////////////////////////////////////////////////////////////////
@@ -121,3 +123,40 @@ void TestAreaStore::genericStoreTest(AreaStore *store)
store->removeArea(d.id);
}

void TestAreaStore::testSerialization()
{
VectorAreaStore store;

Area a(v3s16(-1, 0, 1), v3s16(0, 1, 2));
a.data = "Area A";
store.insertArea(&a);

Area b(v3s16(123, 456, 789), v3s16(32000, 100, 10));
b.data = "Area B";
store.insertArea(&b);

std::ostringstream os;
store.serialize(os);
std::string str = os.str();

std::string str_wanted("\x00" // Version
"\x00\x02" // Count
"\xFF\xFF\x00\x00\x00\x01" // Area A min edge
"\x00\x00\x00\x01\x00\x02" // Area A max edge
"\x00\x06" // Area A data length
"Area A" // Area A data
"\x00\x7B\x00\x64\x00\x0A" // Area B min edge (last two swapped with max edge for sorting)
"\x7D\x00\x01\xC8\x03\x15" // Area B max edge (^)
"\x00\x06" // Area B data length
"Area B", // Area B data
1 + 2 +
6 + 6 + 2 + 6 +
6 + 6 + 2 + 6);
UASSERTEQ(std::string, str, str_wanted);

std::istringstream is(str);
store.deserialize(is);

UASSERTEQ(size_t, store.size(), 4); // deserialize() doesn't clear the store
}

@@ -62,57 +62,42 @@ const Area *AreaStore::getArea(u32 id) const
return &it->second;
}

#if 0
Currently, serialisation is commented out. This is because of multiple reasons:
1. Why do we store the areastore into a file, why not into the database?
2. We don't use libspatial's serialisation, but we should, or perhaps not, because
it would remove the ability to switch. Perhaps write migration routines?
3. Various things need fixing, e.g. the size is serialized as
c++ implementation defined size_t
bool AreaStore::deserialize(std::istream &is)
void AreaStore::serialize(std::ostream &os) const
{
u8 ver = readU8(is);
if (ver != 1)
return false;
u16 count_areas = readU16(is);
for (u16 i = 0; i < count_areas; i++) {
// deserialize an area
Area a;
a.id = readU32(is);
a.minedge = readV3S16(is);
a.maxedge = readV3S16(is);
a.datalen = readU16(is);
a.data = new char[a.datalen];
is.read((char *) a.data, a.datalen);
insertArea(a);
writeU8(os, 0); // Serialisation version

// TODO: Compression?
writeU16(os, areas_map.size());
for (AreaMap::const_iterator it = areas_map.begin();
it != areas_map.end(); ++it) {
const Area &a = it->second;
writeV3S16(os, a.minedge);
writeV3S16(os, a.maxedge);
writeU16(os, a.data.size());
os.write(a.data.data(), a.data.size());
}
return true;
}


static bool serialize_area(void *ostr, Area *a)
void AreaStore::deserialize(std::istream &is)
{
std::ostream &os = *((std::ostream *) ostr);
writeU32(os, a->id);
writeV3S16(os, a->minedge);
writeV3S16(os, a->maxedge);
writeU16(os, a->datalen);
os.write(a->data, a->datalen);

return false;
}

u8 ver = readU8(is);
if (ver != 0)
throw SerializationError("Unknown AreaStore "
"serialization version!");

void AreaStore::serialize(std::ostream &os) const
{
// write initial data
writeU8(os, 1); // serialisation version
writeU16(os, areas_map.size()); //DANGER: not platform independent
forEach(&serialize_area, &os);
u16 num_areas = readU16(is);
for (u32 i = 0; i < num_areas; ++i) {
Area a;
a.minedge = readV3S16(is);
a.maxedge = readV3S16(is);
u16 data_len = readU16(is);
char *data = new char[data_len];
is.read(data, data_len);
a.data = std::string(data, data_len);
insertArea(&a);
}
}

#endif

void AreaStore::invalidateCache()
{
if (m_cache_enabled) {
@@ -226,18 +211,6 @@ void VectorAreaStore::getAreasInArea(std::vector<Area *> *result,
}
}

#if 0
bool SimpleAreaStore::forEach(ForEachCallback callback, void *arg) const
{
for (size_t i = 0; i < m_areas.size(); ++i) {
if (callback(m_areas[i], arg)) {
return true;
}
}
return false;
}
#endif

#if USE_SPATIAL

static inline SpatialIndex::Region get_spatial_region(const v3s16 minedge,
@@ -301,14 +274,6 @@ void SpatialAreaStore::getAreasInArea(std::vector<Area *> *result,
}
}

#if 0
bool SpatialAreaStore::forEach(ForEachCallback callback, void *arg) const
{
// TODO ?? (this is only needed for serialisation, but libspatial has its own serialisation)
return false;
}
#endif

SpatialAreaStore::~SpatialAreaStore()
{
delete m_tree;
@@ -92,16 +92,14 @@ class AreaStore {
/// or NULL if it doesn't exist.
const Area *getArea(u32 id) const;

#if 0
typedef bool (*ForEachCallback)(const Area *a, void *arg);
/// Calls a passed function for every stored area, until the
/// callback returns true. If that happens, it returns true,
/// if the search is exhausted, it returns false.
virtual bool forEach(ForEachCallback, void *arg=NULL) const = 0;

/// Serializes the store's areas to a binary ostream.
void serialize(std::ostream &is) const;
bool deserialize(std::istream &is);
#endif

/// Deserializes the Areas from a binary istream.
/// This does not currently clear the AreaStore before adding the
/// areas, making it possible to deserialize multiple serialized
/// AreaStores.
void deserialize(std::istream &is);

protected:
/// Invalidates the getAreasForPos cache.
@@ -141,7 +139,6 @@ class VectorAreaStore : public AreaStore {
virtual bool removeArea(u32 id);
virtual void getAreasInArea(std::vector<Area *> *result,
v3s16 minedge, v3s16 maxedge, bool accept_overlap);
//virtual bool forEach(ForEachCallback, void *arg) const;

protected:
virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);
@@ -162,7 +159,6 @@ class SpatialAreaStore : public AreaStore {
virtual bool removeArea(u32 id);
virtual void getAreasInArea(std::vector<Area *> *result,
v3s16 minedge, v3s16 maxedge, bool accept_overlap);
//virtual bool forEach(ForEachCallback, void *arg) const;

protected:
virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);

0 comments on commit 821551a

Please sign in to comment.