Permalink
Browse files

some cleanup

  • Loading branch information...
albertz committed Mar 7, 2011
1 parent 00a2f14 commit 1467a5892a080bc51a00f2e234da43bf6e57379d
Showing with 105 additions and 82 deletions.
  1. +1 −0 Db.h
  2. +1 −0 DbFsBackend.h
  3. +28 −38 DbKyotoBackend.cpp
  4. +4 −2 DbKyotoBackend.h
  5. +6 −2 DbRedisBackend.cpp
  6. +5 −1 DbRedisBackend.h
  7. +9 −3 db-list-dir.cpp
  8. +25 −14 db-push-dir.cpp
  9. +26 −22 db-push.cpp
View
1 Db.h
@@ -64,6 +64,7 @@ struct DbDirEntry {
struct DbIntf {
DbStats stats;
+ virtual Return init() = 0;
virtual Return push(/*out*/ DbEntryId& id, const DbEntry& entry) = 0;
virtual Return get(/*out*/ DbEntry& entry, const DbEntryId& id) = 0;
virtual Return pushToDir(const std::string& path, const DbDirEntry& dirEntry) {
View
@@ -12,6 +12,7 @@ struct DbFsBackend : DbIntf {
std::string baseDir;
DbFsBackend(const std::string& d = "db") : baseDir(d) {}
+ Return init() { return true; }
Return push(/*out*/ DbEntryId& id, const DbEntry& entry);
Return get(/*out*/ DbEntry& entry, const DbEntryId& id);
};
View
@@ -10,14 +10,18 @@
#include <iostream>
using namespace std;
+using namespace kyotocabinet;
-DbKyotoBackend::DbKyotoBackend(const std::string& _prefix) {
- prefix = _prefix;
+DbKyotoBackend::~DbKyotoBackend() {
}
-DbRedisBackend::~DbRedisBackend() {
+Return DbKyotoBackend::init() {
+ if(!db.open(filename, PolyDB::OWRITER | PolyDB::OCREATE))
+ return std::string() + "failed to open KyotoCabinet DB: " + db.error().name();
+ return true;
}
+/*
static Return __saveNewDbEntry(redisContext* redis, const std::string& prefix, DbEntryId& id, const std::string& content) {
unsigned short triesNum = (id.size() <= 4) ? (2 << id.size()) : 64;
for(unsigned short i = 0; i < triesNum; ++i) {
@@ -35,21 +39,17 @@ static Return __saveNewDbEntry(redisContext* redis, const std::string& prefix, D
id += (char)random();
return __saveNewDbEntry(redis, prefix, id, content);
}
+ */
Return DbKyotoBackend::push(/*out*/ DbEntryId& id, const DbEntry& entry) {
if(!entry.haveSha1())
return "DB push: entry SHA1 not calculated";
if(!entry.haveCompressed())
return "DB push: entry compression not calculated";
- if(redis == NULL)
- return "DB push: Redis connection not initialized";
- if(!(redis->flags & REDIS_CONNECTED))
- return "DB push: Redis not connected";
// search for existing entry
- std::string sha1refkey = prefix + "sha1ref." + entry.sha1;
- RedisReplyWrapper reply( redisCommand(redis, "SMEMBERS %b", &sha1refkey[0], sha1refkey.size()) );
- ASSERT( reply );
+ std::string sha1refkey = "sha1ref." + entry.sha1;
+/* RedisReplyWrapper reply( redisCommand(redis, "SMEMBERS %b", &sha1refkey[0], sha1refkey.size()) );
if(reply.reply->type == REDIS_REPLY_ARRAY)
for(int i = 0; i < reply.reply->elements; ++i) {
DbEntryId otherId = std::string(reply.reply->element[i]->str, reply.reply->element[i]->len);
@@ -63,77 +63,67 @@ Return DbKyotoBackend::push(/*out*/ DbEntryId& id, const DbEntry& entry) {
}
}
}
-
+*/
// write DB entry
id = "";
- ASSERT( __saveNewDbEntry(redis, prefix, id, entry.compressed) );
+// ASSERT( __saveNewDbEntry(redis, prefix, id, entry.compressed) );
// create sha1 ref
- reply = redisCommand(redis, "SADD %b %b", &sha1refkey[0], sha1refkey.size(), &id[0], id.size());
- ASSERT( reply );
+// reply = redisCommand(redis, "SADD %b %b", &sha1refkey[0], sha1refkey.size(), &id[0], id.size());
+// ASSERT( reply );
stats.pushNew++;
return true;
}
Return DbKyotoBackend::get(/*out*/ DbEntry& entry, const DbEntryId& id) {
- if(redis == NULL)
- return "DB get: Redis connection not initialized";
- if(!(redis->flags & REDIS_CONNECTED))
- return "DB get: Redis not connected";
-
- std::string key = prefix + "data." + id;
- RedisReplyWrapper reply( redisCommand(redis, "GET %b", &key[0], key.size()) );
- ASSERT(reply);
+ std::string key = "data." + id;
+/* RedisReplyWrapper reply( redisCommand(redis, "GET %b", &key[0], key.size()) );
if(reply.reply->type == REDIS_REPLY_NIL)
return "DB get: entry not found";
if(reply.reply->type != REDIS_REPLY_STRING)
return "DB get: Redis: invalid GET reply";
- entry.compressed = std::string(reply.reply->str, reply.reply->len);
+ entry.compressed = std::string(reply.reply->str, reply.reply->len); */
- ASSERT( entry.uncompress() );
+ //ASSERT( entry.uncompress() );
entry.calcSha1();
return true;
}
Return DbKyotoBackend::pushToDir(const std::string& path, const DbDirEntry& dirEntry) {
- std::string key = prefix + "fs." + path;
+ std::string key = "fs." + path;
std::string dirEntryRaw = dirEntry.serialized();
- RedisReplyWrapper reply( redisCommand(redis, "SADD %b %b", &key[0], key.size(), &dirEntryRaw[0], dirEntryRaw.size()) );
- ASSERT( reply );
+// RedisReplyWrapper reply( redisCommand(redis, "SADD %b %b", &key[0], key.size(), &dirEntryRaw[0], dirEntryRaw.size()) );
return true;
}
Return DbKyotoBackend::getDir(/*out*/ std::list<DbDirEntry>& dirList, const std::string& path) {
- std::string key = prefix + "fs." + path;
- RedisReplyWrapper reply( redisCommand(redis, "SMEMBERS %b", &key[0], key.size()) );
- ASSERT( reply );
+ std::string key = "fs." + path;
+/* RedisReplyWrapper reply( redisCommand(redis, "SMEMBERS %b", &key[0], key.size()) );
if(reply.reply->type != REDIS_REPLY_ARRAY)
return "DB getDir: invalid SMEMBERS reply";
for(int i = 0; i < reply.reply->elements; ++i) {
std::string dirEntryRaw(reply.reply->element[i]->str, reply.reply->element[i]->len);
dirList.push_back( DbDirEntry::FromSerialized(dirEntryRaw) );
- }
+ }*/
return true;
}
Return DbKyotoBackend::setFileRef(/*can be empty*/ const DbEntryId& id, const std::string& path) {
- std::string key = prefix + "fs." + path;
- RedisReplyWrapper reply( redisCommand(redis, "SET %b %b", &key[0], key.size(), &id[0], id.size()) );
- ASSERT( reply );
+ std::string key = "fs." + path;
+// RedisReplyWrapper reply( redisCommand(redis, "SET %b %b", &key[0], key.size(), &id[0], id.size()) );
return true;
}
Return DbKyotoBackend::getFileRef(/*out (can be empty)*/ DbEntryId& id, const std::string& path) {
- std::string key = prefix + "fs." + path;
- RedisReplyWrapper reply( redisCommand(redis, "GET %b", &key[0], key.size()) );
- ASSERT( reply );
+ std::string key = "fs." + path;
+/* RedisReplyWrapper reply( redisCommand(redis, "GET %b", &key[0], key.size()) );
if(reply.reply->type != REDIS_REPLY_STRING)
return "DB getFileRef: invalid GET reply";
- id = std::string(reply.reply->str, reply.reply->len);
+ id = std::string(reply.reply->str, reply.reply->len); */
return true;
}
View
@@ -10,10 +10,12 @@
#include "kyotocabinet/kcpolydb.h"
struct DbKyotoBackend : DbIntf {
- std::string prefix;
+ kyotocabinet::PolyDB db;
+ std::string filename;
- DbKyotoBackend(const std::string& prefix = "db.");
+ DbKyotoBackend(const std::string& dbfilename = "db.kch") : filename(dbfilename) {}
~DbKyotoBackend();
+ Return init();
Return push(/*out*/ DbEntryId& id, const DbEntry& entry);
Return get(/*out*/ DbEntry& entry, const DbEntryId& id);
Return pushToDir(const std::string& path, const DbDirEntry& dirEntry);
View
@@ -11,9 +11,13 @@
#include <iostream>
using namespace std;
-DbRedisBackend::DbRedisBackend(const std::string& _prefix, const std::string& host, int port) {
- prefix = _prefix;
+Return DbRedisBackend::init() {
redis = redisConnect(host.c_str(), port);
+ if(redis == NULL)
+ return "failed to init Redis";
+ if(!(redis->flags & REDIS_CONNECTED))
+ return "failed to connect to Redis server";
+ return true;
}
DbRedisBackend::~DbRedisBackend() {
View
@@ -11,10 +11,14 @@
struct DbRedisBackend : DbIntf {
std::string prefix;
+ std::string host;
+ int port;
redisContext* redis;
- DbRedisBackend(const std::string& prefix = "db.", const std::string& host = "127.0.0.1", int port = 6379);
+ DbRedisBackend(const std::string& _prefix = "db.", const std::string& _host = "127.0.0.1", int _port = 6379)
+ : prefix(_prefix), host(_host), port(_port), redis(NULL) {}
~DbRedisBackend();
+ Return init();
Return push(/*out*/ DbEntryId& id, const DbEntry& entry);
Return get(/*out*/ DbEntry& entry, const DbEntryId& id);
Return pushToDir(const std::string& path, const DbDirEntry& dirEntry);
View
@@ -48,12 +48,18 @@ static Return listDir(const std::string& path) {
return true;
}
+static Return _main() {
+ DbDefBackend dbInst;
+ db = &dbInst;
+ ASSERT( db->init() );
+ ASSERT( listDir("") );
+ return true;
+}
+
int main(int argc, char** argv) {
srandom(time(NULL));
- DbDefBackend dbInst;
- db = &dbInst;
- Return r = listDir("");
+ Return r = _main();
if(!r) {
cerr << "error: " << r.errmsg << endl;
return 1;
View
@@ -15,21 +15,14 @@
#include <iostream>
using namespace std;
-int main(int argc, char** argv) {
- if(argc <= 1) {
- cerr << "please give me a dirname" << endl;
- return 1;
- }
-
- srandom(time(NULL));
+Return _main(const std::string& dirname) {
DbDefBackend db;
-
- std::string dirname = argv[1];
+ ASSERT( db.init() );
+
DirIter dir(dirname);
- if(dir.dir == NULL) {
- cerr << "error: cannot open directory " << dirname << endl;
- return 1;
- }
+ if(dir.dir == NULL)
+ return "cannot open directory " + dirname;
+
for(; dir; dir.next()) {
if(dir.filename.size() <= 4) continue;
if(dir.filename.substr(dir.filename.size()-4) != ".png") continue;
@@ -58,7 +51,25 @@ int main(int argc, char** argv) {
<< db.stats.pushReuse << " / " << db.stats.pushNew
<< endl;
}
-
+
+ return true;
+}
+
+int main(int argc, char** argv) {
+ if(argc <= 1) {
+ cerr << "please give me a dirname" << endl;
+ return 1;
+ }
+
+ srandom(time(NULL));
+
+ std::string dirname = argv[1];
+ Return r = _main(dirname);
+ if(!r) {
+ cerr << "error: " << r.errmsg << endl;
+ return 1;
+ }
+
cout << "success" << endl;
return 0;
}
View
@@ -14,39 +14,43 @@
#include <iostream>
using namespace std;
-int main(int argc, char** argv) {
- if(argc <= 1) {
- cerr << "please give me a filename" << endl;
- return 1;
- }
-
- std::string filename = argv[1];
+Return _main(const std::string& filename) {
FILE* f = fopen(filename.c_str(), "rb");
- if(f == NULL) {
- cerr << "error: cannot open " << filename << endl;
- return 1;
- }
+ if(f == NULL)
+ return "cannot open " + filename;
- srandom(time(NULL));
DbDefBackend db;
+ ASSERT( db.init() );
DbPngEntryWriter dbPngWriter(f, &db);
- while(dbPngWriter) {
- Return r = dbPngWriter.next();
- if(!r) {
- cerr << "error: " << r.errmsg << endl;
- return 1;
- }
- }
+ while(dbPngWriter)
+ ASSERT( dbPngWriter.next() );
- db.pushToDir("", DbDirEntry::File(baseFilename(filename), ftell(f)));
- db.setFileRef(dbPngWriter.contentId, "/" + baseFilename(filename));
+ ASSERT( db.pushToDir("", DbDirEntry::File(baseFilename(filename), ftell(f))) );
+ ASSERT( db.setFileRef(dbPngWriter.contentId, "/" + baseFilename(filename)) );
fclose(f);
-
+
cout << "content id: " << hexString(dbPngWriter.contentId) << endl;
cout << "num content entries: " << dbPngWriter.contentEntries.size() << endl;
cout << "db stats: push new: " << db.stats.pushNew << endl;
cout << "db stats: push reuse: " << db.stats.pushReuse << endl;
+ return true;
+}
+
+int main(int argc, char** argv) {
+ if(argc <= 1) {
+ cerr << "please give me a filename" << endl;
+ return 1;
+ }
+
+ std::string filename = argv[1];
+ srandom(time(NULL));
+ Return r = _main(filename);
+ if(!r) {
+ cerr << "error: " << r.errmsg << endl;
+ return 1;
+ }
+
cout << "success" << endl;
return 0;
}

0 comments on commit 1467a58

Please sign in to comment.