126 changes: 58 additions & 68 deletions src/database-redis.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,84 +20,78 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "config.h"

#if USE_REDIS
/*
Redis databases
*/


#include "database-redis.h"
#include <hiredis.h>

#include "map.h"
#include "mapsector.h"
#include "mapblock.h"
#include "serialization.h"
#include "main.h"
#include "settings.h"
#include "log.h"
#include "filesys.h"
#include "exceptions.h"
#include "util/string.h"

#include <hiredis.h>
#include <cassert>


Database_Redis::Database_Redis(ServerMap *map, std::string savedir)
Database_Redis::Database_Redis(Settings &conf)
{
Settings conf;
conf.readConfigFile((std::string(savedir) + DIR_DELIM + "world.mt").c_str());
std::string tmp;
try {
tmp = conf.get("redis_address");
hash = conf.get("redis_hash");
} catch(SettingNotFoundException e) {
throw SettingNotFoundException("Set redis_address and redis_hash in world.mt to use the redis backend");
tmp = conf.get("redis_address");
hash = conf.get("redis_hash");
} catch (SettingNotFoundException) {
throw SettingNotFoundException("Set redis_address and "
"redis_hash in world.mt to use the redis backend");
}
const char *addr = tmp.c_str();
int port = conf.exists("redis_port") ? conf.getU16("redis_port") : 6379;
ctx = redisConnect(addr, port);
if(!ctx)
if (!ctx) {
throw FileNotGoodException("Cannot allocate redis context");
else if(ctx->err) {
} else if(ctx->err) {
std::string err = std::string("Connection error: ") + ctx->errstr;
redisFree(ctx);
throw FileNotGoodException(err);
}
srvmap = map;
}

int Database_Redis::Initialized(void)
Database_Redis::~Database_Redis()
{
return 1;
redisFree(ctx);
}

void Database_Redis::beginSave() {
redisReply *reply;
reply = (redisReply*) redisCommand(ctx, "MULTI");
if(!reply)
throw FileNotGoodException(std::string("redis command 'MULTI' failed: ") + ctx->errstr);
redisReply *reply = static_cast<redisReply *>(redisCommand(ctx, "MULTI"));
if (!reply) {
throw FileNotGoodException(std::string(
"Redis command 'MULTI' failed: ") + ctx->errstr);
}
freeReplyObject(reply);
}

void Database_Redis::endSave() {
redisReply *reply;
reply = (redisReply*) redisCommand(ctx, "EXEC");
if(!reply)
throw FileNotGoodException(std::string("redis command 'EXEC' failed: ") + ctx->errstr);
redisReply *reply = static_cast<redisReply *>(redisCommand(ctx, "EXEC"));
if (!reply) {
throw FileNotGoodException(std::string(
"Redis command 'EXEC' failed: ") + ctx->errstr);
}
freeReplyObject(reply);
}

bool Database_Redis::saveBlock(v3s16 blockpos, std::string &data)
bool Database_Redis::saveBlock(const v3s16 &pos, const std::string &data)
{
std::string tmp = i64tos(getBlockAsInteger(blockpos));
std::string tmp = i64tos(getBlockAsInteger(pos));

redisReply *reply = (redisReply *)redisCommand(ctx, "HSET %s %s %b",
hash.c_str(), tmp.c_str(), data.c_str(), data.size());
redisReply *reply = static_cast<redisReply *>(redisCommand(ctx, "HSET %s %s %b",
hash.c_str(), tmp.c_str(), data.c_str(), data.size()));
if (!reply) {
errorstream << "WARNING: saveBlock: redis command 'HSET' failed on "
"block " << PP(blockpos) << ": " << ctx->errstr << std::endl;
"block " << PP(pos) << ": " << ctx->errstr << std::endl;
freeReplyObject(reply);
return false;
}

if (reply->type == REDIS_REPLY_ERROR) {
errorstream << "WARNING: saveBlock: saving block " << PP(blockpos)
errorstream << "WARNING: saveBlock: saving block " << PP(pos)
<< "failed" << std::endl;
freeReplyObject(reply);
return false;
Expand All @@ -107,38 +101,36 @@ bool Database_Redis::saveBlock(v3s16 blockpos, std::string &data)
return true;
}

std::string Database_Redis::loadBlock(v3s16 blockpos)
std::string Database_Redis::loadBlock(const v3s16 &pos)
{
std::string tmp = i64tos(getBlockAsInteger(blockpos));
redisReply *reply;
reply = (redisReply*) redisCommand(ctx, "HGET %s %s", hash.c_str(), tmp.c_str());
std::string tmp = i64tos(getBlockAsInteger(pos));
redisReply *reply = static_cast<redisReply *>(redisCommand(ctx,
"HGET %s %s", hash.c_str(), tmp.c_str()));

if(!reply)
throw FileNotGoodException(std::string("redis command 'HGET %s %s' failed: ") + ctx->errstr);
if(reply->type != REDIS_REPLY_STRING)
if (!reply) {
throw FileNotGoodException(std::string(
"Redis command 'HGET %s %s' failed: ") + ctx->errstr);
} else if (reply->type != REDIS_REPLY_STRING) {
return "";
}

std::string str(reply->str, reply->len);
freeReplyObject(reply); // std::string copies the memory so this won't cause any problems
return str;
}

bool Database_Redis::deleteBlock(v3s16 blockpos)
bool Database_Redis::deleteBlock(const v3s16 &pos)
{
std::string tmp = i64tos(getBlockAsInteger(blockpos));
std::string tmp = i64tos(getBlockAsInteger(pos));

redisReply *reply = (redisReply *)redisCommand(ctx, "HDEL %s %s",
hash.c_str(), tmp.c_str());
redisReply *reply = static_cast<redisReply *>(redisCommand(ctx,
"HDEL %s %s", hash.c_str(), tmp.c_str()));
if (!reply) {
errorstream << "WARNING: deleteBlock: redis command 'HDEL' failed on "
"block " << PP(blockpos) << ": " << ctx->errstr << std::endl;
freeReplyObject(reply);
return false;
}

if (reply->type == REDIS_REPLY_ERROR) {
errorstream << "WARNING: deleteBlock: deleting block " << PP(blockpos)
<< "failed" << std::endl;
throw FileNotGoodException(std::string(
"Redis command 'HDEL %s %s' failed: ") + ctx->errstr);
} else if (reply->type == REDIS_REPLY_ERROR) {
errorstream << "WARNING: deleteBlock: deleting block " << PP(pos)
<< " failed" << std::endl;
freeReplyObject(reply);
return false;
}
Expand All @@ -149,21 +141,19 @@ bool Database_Redis::deleteBlock(v3s16 blockpos)

void Database_Redis::listAllLoadableBlocks(std::vector<v3s16> &dst)
{
redisReply *reply;
reply = (redisReply*) redisCommand(ctx, "HKEYS %s", hash.c_str());
if(!reply)
throw FileNotGoodException(std::string("redis command 'HKEYS %s' failed: ") + ctx->errstr);
if(reply->type != REDIS_REPLY_ARRAY)
redisReply *reply = static_cast<redisReply *>(redisCommand(ctx, "HKEYS %s", hash.c_str()));
if (!reply) {
throw FileNotGoodException(std::string(
"Redis command 'HKEYS %s' failed: ") + ctx->errstr);
} else if (reply->type != REDIS_REPLY_ARRAY) {
throw FileNotGoodException("Failed to get keys from database");
for(size_t i = 0; i < reply->elements; i++) {
}
for (size_t i = 0; i < reply->elements; i++) {
assert(reply->element[i]->type == REDIS_REPLY_STRING);
dst.push_back(getIntegerAsBlock(stoi64(reply->element[i]->str)));
}
freeReplyObject(reply);
}

Database_Redis::~Database_Redis()
{
redisFree(ctx);
}
#endif
#endif // USE_REDIS

22 changes: 13 additions & 9 deletions src/database-redis.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,28 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <hiredis.h>
#include <string>

class ServerMap;
class Settings;

class Database_Redis : public Database
{
public:
Database_Redis(ServerMap *map, std::string savedir);
Database_Redis(Settings &conf);
~Database_Redis();

virtual void beginSave();
virtual void endSave();
virtual bool saveBlock(v3s16 blockpos, std::string &data);
virtual std::string loadBlock(v3s16 blockpos);
virtual bool deleteBlock(v3s16 blockpos);

virtual bool saveBlock(const v3s16 &pos, const std::string &data);
virtual std::string loadBlock(const v3s16 &pos);
virtual bool deleteBlock(const v3s16 &pos);
virtual void listAllLoadableBlocks(std::vector<v3s16> &dst);
virtual int Initialized(void);
~Database_Redis();

private:
ServerMap *srvmap;
redisContext *ctx;
std::string hash;
};

#endif // USE_REDIS

#endif
#endif

337 changes: 136 additions & 201 deletions src/database-sqlite3.cpp

Large diffs are not rendered by default.

42 changes: 25 additions & 17 deletions src/database-sqlite3.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,35 +27,43 @@ extern "C" {
#include "sqlite3.h"
}

class ServerMap;

class Database_SQLite3 : public Database
{
public:
Database_SQLite3(ServerMap *map, std::string savedir);
Database_SQLite3(const std::string &savedir);

virtual void beginSave();
virtual void endSave();

virtual bool saveBlock(v3s16 blockpos, std::string &data);
virtual std::string loadBlock(v3s16 blockpos);
virtual bool deleteBlock(v3s16 blockpos);
virtual bool saveBlock(const v3s16 &pos, const std::string &data);
virtual std::string loadBlock(const v3s16 &pos);
virtual bool deleteBlock(const v3s16 &pos);
virtual void listAllLoadableBlocks(std::vector<v3s16> &dst);
virtual int Initialized(void);
virtual bool initialized() const { return m_initialized; }
~Database_SQLite3();
private:
ServerMap *srvmap;
std::string m_savedir;
sqlite3 *m_database;
sqlite3_stmt *m_database_read;
sqlite3_stmt *m_database_write;
sqlite3_stmt *m_database_delete;
sqlite3_stmt *m_database_list;

private:
// Open the database
void openDatabase();
// Create the database structure
void createDatabase();
// Verify we can read/write to the database
// Open and initialize the database if needed
void verifyDatabase();
void createDirs(std::string path);

void bindPos(sqlite3_stmt *stmt, const v3s16 &pos, int index=1);

bool m_initialized;

std::string m_savedir;

sqlite3 *m_database;
sqlite3_stmt *m_stmt_read;
sqlite3_stmt *m_stmt_write;
sqlite3_stmt *m_stmt_list;
sqlite3_stmt *m_stmt_delete;
sqlite3_stmt *m_stmt_begin;
sqlite3_stmt *m_stmt_end;
};

#endif

4 changes: 2 additions & 2 deletions src/database.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,15 @@ static inline s64 pythonmodulo(s64 i, s16 mod)
}


s64 Database::getBlockAsInteger(const v3s16 pos) const
s64 Database::getBlockAsInteger(const v3s16 &pos)
{
return (u64) pos.Z * 0x1000000 +
(u64) pos.Y * 0x1000 +
(u64) pos.X;
}


v3s16 Database::getIntegerAsBlock(s64 i) const
v3s16 Database::getIntegerAsBlock(s64 i)
{
v3s16 pos;
pos.X = unsigned_to_signed(pythonmodulo(i, 4096), 2048);
Expand Down
26 changes: 16 additions & 10 deletions src/database.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,22 @@ with this program; if not, write to the Free Software Foundation, Inc.,
class Database
{
public:
virtual void beginSave() = 0;
virtual void endSave() = 0;

virtual bool saveBlock(v3s16 blockpos, std::string &data) = 0;
virtual std::string loadBlock(v3s16 blockpos) = 0;
virtual bool deleteBlock(v3s16 blockpos) = 0;
s64 getBlockAsInteger(const v3s16 pos) const;
v3s16 getIntegerAsBlock(s64 i) const;
virtual ~Database() {}

virtual void beginSave() {}
virtual void endSave() {}

virtual bool saveBlock(const v3s16 &pos, const std::string &data) = 0;
virtual std::string loadBlock(const v3s16 &pos) = 0;
virtual bool deleteBlock(const v3s16 &pos) = 0;

static s64 getBlockAsInteger(const v3s16 &pos);
static v3s16 getIntegerAsBlock(s64 i);

virtual void listAllLoadableBlocks(std::vector<v3s16> &dst) = 0;
virtual int Initialized(void)=0;
virtual ~Database() {};

virtual bool initialized() const { return true; }
};

#endif

59 changes: 26 additions & 33 deletions src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -914,52 +914,46 @@ static bool run_dedicated_server(const GameParams &game_params, const Settings &
static bool migrate_database(const GameParams &game_params, const Settings &cmd_args,
Server *server)
{
std::string migrate_to = cmd_args.get("migrate");
Settings world_mt;
bool success = world_mt.readConfigFile((game_params.world_path
+ DIR_DELIM + "world.mt").c_str());
std::string world_mt_path = game_params.world_path + DIR_DELIM + "world.mt";
bool success = world_mt.readConfigFile(world_mt_path.c_str());
if (!success) {
errorstream << "Cannot read world.mt" << std::endl;
return false;
return 1;
}

if (!world_mt.exists("backend")) {
errorstream << "Please specify your current backend in world.mt file:"
<< std::endl << " backend = {sqlite3|leveldb|redis|dummy}"
<< std::endl;
return false;
errorstream << "Please specify your current backend in world.mt:"
<< std::endl;
errorstream << " backend = {sqlite3|leveldb|redis|dummy}"
<< std::endl;
return 1;
}

std::string backend = world_mt.get("backend");
Database *new_db;
std::string migrate_to = cmd_args.get("migrate");

if (backend == migrate_to) {
errorstream << "Cannot migrate: new backend is same as the old one"
<< std::endl;
return false;
errorstream << "Cannot migrate: new backend is same"
<<" as the old one" << std::endl;
return 1;
}

if (migrate_to == "sqlite3")
new_db = new Database_SQLite3(&(ServerMap&)server->getMap(),
game_params.world_path);
#if USE_LEVELDB
new_db = new Database_SQLite3(game_params.world_path);
#if USE_LEVELDB
else if (migrate_to == "leveldb")
new_db = new Database_LevelDB(&(ServerMap&)server->getMap(),
game_params.world_path);
#endif
#if USE_REDIS
new_db = new Database_LevelDB(game_params.world_path);
#endif
#if USE_REDIS
else if (migrate_to == "redis")
new_db = new Database_Redis(&(ServerMap&)server->getMap(),
game_params.world_path);
#endif
new_db = new Database_Redis(world_mt);
#endif
else {
errorstream << "Migration to " << migrate_to << " is not supported"
<< std::endl;
return false;
errorstream << "Migration to " << migrate_to
<< " is not supported" << std::endl;
return 1;
}

std::vector<v3s16> blocks;
ServerMap &old_map = ((ServerMap&)server->getMap());
ServerMap &old_map = (ServerMap &) server->getMap();
old_map.listAllLoadableBlocks(blocks);
int count = 0;
new_db->beginSave();
Expand All @@ -975,16 +969,15 @@ static bool migrate_database(const GameParams &game_params, const Settings &cmd_
}
++count;
if (count % 500 == 0)
actionstream << "Migrated " << count << " blocks "
<< (100.0 * count / blocks.size()) << "% completed" << std::endl;
actionstream << "Migrated " << count << " blocks "
<< (100.0 * count / blocks.size()) << "% completed" << std::endl;
}
new_db->endSave();
delete new_db;

actionstream << "Successfully migrated " << count << " blocks" << std::endl;
world_mt.set("backend", migrate_to);
if (!world_mt.updateConfigFile(
(game_params.world_path+ DIR_DELIM + "world.mt").c_str()))
if (!world_mt.updateConfigFile(world_mt_path.c_str()))
errorstream << "Failed to update world.mt!" << std::endl;
else
actionstream << "world.mt updated" << std::endl;
Expand Down
76 changes: 31 additions & 45 deletions src/map.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,22 +53,6 @@ with this program; if not, write to the Free Software Foundation, Inc.,

#define PP(x) "("<<(x).X<<","<<(x).Y<<","<<(x).Z<<")"

/*
SQLite format specification:
- Initially only replaces sectors/ and sectors2/
If map.sqlite does not exist in the save dir
or the block was not found in the database
the map will try to load from sectors folder.
In either case, map.sqlite will be created
and all future saves will save there.
Structure of map.sqlite:
Tables:
blocks
(PK) INT pos
BLOB data
*/

/*
Map
Expand Down Expand Up @@ -2031,25 +2015,26 @@ ServerMap::ServerMap(std::string savedir, IGameDef *gamedef, EmergeManager *emer
bool succeeded = conf.readConfigFile(conf_path.c_str());
if (!succeeded || !conf.exists("backend")) {
// fall back to sqlite3
dbase = new Database_SQLite3(this, savedir);
conf.set("backend", "sqlite3");
} else {
std::string backend = conf.get("backend");
if (backend == "dummy")
dbase = new Database_Dummy(this);
else if (backend == "sqlite3")
dbase = new Database_SQLite3(this, savedir);
#if USE_LEVELDB
else if (backend == "leveldb")
dbase = new Database_LevelDB(this, savedir);
#endif
#if USE_REDIS
else if (backend == "redis")
dbase = new Database_Redis(this, savedir);
#endif
else
throw BaseException("Unknown map backend");
}
std::string backend = conf.get("backend");
if (backend == "dummy")
dbase = new Database_Dummy();
else if (backend == "sqlite3")
dbase = new Database_SQLite3(savedir);
#if USE_LEVELDB
else if (backend == "leveldb")
dbase = new Database_LevelDB(savedir);
#endif
#if USE_REDIS
else if (backend == "redis")
dbase = new Database_Redis(conf);
#endif
else
throw BaseException("Unknown map backend");

if (!conf.updateConfigFile(conf_path.c_str()))
errorstream << "ServerMap::ServerMap(): Failed to update world.mt!" << std::endl;

m_savedir = savedir;
m_map_saving_enabled = false;
Expand Down Expand Up @@ -2828,7 +2813,8 @@ s16 ServerMap::findGroundLevel(v2s16 p2d)
}

bool ServerMap::loadFromFolders() {
if(!dbase->Initialized() && !fs::PathExists(m_savedir + DIR_DELIM + "map.sqlite")) // ?
if(!dbase->initialized() &&
!fs::PathExists(m_savedir + DIR_DELIM + "map.sqlite"))
return true;
return false;
}
Expand All @@ -2850,14 +2836,14 @@ std::string ServerMap::getSectorDir(v2s16 pos, int layout)
{
case 1:
snprintf(cc, 9, "%.4x%.4x",
(unsigned int)pos.X&0xffff,
(unsigned int)pos.Y&0xffff);
(unsigned int) pos.X & 0xffff,
(unsigned int) pos.Y & 0xffff);

return m_savedir + DIR_DELIM + "sectors" + DIR_DELIM + cc;
case 2:
snprintf(cc, 9, "%.3x" DIR_DELIM "%.3x",
(unsigned int)pos.X&0xfff,
(unsigned int)pos.Y&0xfff);
snprintf(cc, 9, (std::string("%.3x") + DIR_DELIM + "%.3x").c_str(),
(unsigned int) pos.X & 0xfff,
(unsigned int) pos.Y & 0xfff);

return m_savedir + DIR_DELIM + "sectors2" + DIR_DELIM + cc;
default:
Expand All @@ -2881,10 +2867,10 @@ v2s16 ServerMap::getSectorPos(std::string dirname)
{
// New layout
fs::RemoveLastPathComponent(dirname, &component, 2);
r = sscanf(component.c_str(), "%3x" DIR_DELIM "%3x", &x, &y);
r = sscanf(component.c_str(), (std::string("%3x") + DIR_DELIM + "%3x").c_str(), &x, &y);
// Sign-extend the 12 bit values up to 16 bits...
if(x&0x800) x|=0xF000;
if(y&0x800) y|=0xF000;
if(x & 0x800) x |= 0xF000;
if(y & 0x800) y |= 0xF000;
}
else
{
Expand Down Expand Up @@ -3299,7 +3285,7 @@ bool ServerMap::saveBlock(MapBlock *block, Database *db)

std::string data = o.str();
bool ret = db->saveBlock(p3d, data);
if(ret) {
if (ret) {
// We just wrote it to the disk so clear modified flag
block->resetModified();
}
Expand All @@ -3311,7 +3297,7 @@ void ServerMap::loadBlock(std::string sectordir, std::string blockfile,
{
DSTACK(__FUNCTION_NAME);

std::string fullpath = sectordir+DIR_DELIM+blockfile;
std::string fullpath = sectordir + DIR_DELIM + blockfile;
try {

std::ifstream is(fullpath.c_str(), std::ios_base::binary);
Expand Down Expand Up @@ -3513,7 +3499,7 @@ MapBlock* ServerMap::loadBlock(v3s16 blockpos)
*/

std::string blockfilename = getBlockFilename(blockpos);
if(fs::PathExists(sectordir+DIR_DELIM+blockfilename) == false)
if(fs::PathExists(sectordir + DIR_DELIM + blockfilename) == false)
return NULL;

/*
Expand Down
2 changes: 1 addition & 1 deletion src/map.h
Original file line number Diff line number Diff line change
Expand Up @@ -481,8 +481,8 @@ class ServerMap : public Map
// Returns true if sector now resides in memory
//bool deFlushSector(v2s16 p2d);

bool saveBlock(MapBlock *block, Database *db);
bool saveBlock(MapBlock *block);
static bool saveBlock(MapBlock *block, Database *db);
// This will generate a sector with getSector if not found.
void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load=false);
MapBlock* loadBlock(v3s16 p);
Expand Down
4 changes: 2 additions & 2 deletions src/network/packethandlers/client.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,8 @@ void Client::handleCommand_BlockData(NetworkPacket* pkt)
sector->insertBlock(block);
}

if (localdb != NULL) {
((ServerMap&) localserver->getMap()).saveBlock(block, localdb);
if (m_localdb) {
ServerMap::saveBlock(block, m_localdb);
}

/*
Expand Down