Skip to content

Commit

Permalink
Revert "Revert "Deploy lazy-sync for sandwich db""
Browse files Browse the repository at this point in the history
:Release Notes:
Revert "Deploy lazy-sync for sandwich db" for more accurate testing and deploy

:Detailed Notes:
This reverts commit a728677.

:Testing Performed:
Compiled and tested localy.

:QA Notes:

:Issues Addressed:
[CMB-39] Proposal lazy sync for improving writing performance

Open-webOS-DCO-1.0-Signed-off-by: Maksym Sditanov <maxim.sditanov@lge.com>

Change-Id: Iaba4e939065b7427e8b64406af268eab8fa1421c
Reviewed-on: https://g2g.lgsvl.com/6509
Tested-by: Maksym Sditanov <maxim.sditanov@lge.com>
Reviewed-by: Olexandr Solomakha <oleksander.solomakha@lge.com>
Reviewed-by: Maksym Sditanov <maxim.sditanov@lge.com>
  • Loading branch information
Maksym Sditanov committed Sep 8, 2014
1 parent a728677 commit 1a90b24
Show file tree
Hide file tree
Showing 19 changed files with 359 additions and 10 deletions.
10 changes: 10 additions & 0 deletions files/conf/maindb.conf
Expand Up @@ -32,6 +32,16 @@
"maxLockers" : 1000,
"compactStepSize" : 25000
},
"sandwich" : {
"cacheSize": 3145728,
"maxLocks" : 20000,
"maxLockers" : 1000,
"compactStepSize" : 25000,
"sync" : 1,
"paranoid_checks" : 1,
"verify_checksums" : 0,
"fill_cache" : 1
},
"ldb" : {
"cacheSize": 3145728,
"maxLocks" : 20000,
Expand Down
1 change: 1 addition & 0 deletions inc/db/MojDbReq.h
Expand Up @@ -65,6 +65,7 @@ class MojDbReq : private MojNoCopy
MojInt32 batchsize() {return m_batchSize;}
operator MojDbReqRef() { return MojDbReqRef(*this); }

bool schemaLocked() const { return m_schemaLocked; }


private:
Expand Down
1 change: 1 addition & 0 deletions src/db-luna/BackendBuildRules.cmake
Expand Up @@ -107,6 +107,7 @@ foreach (backend ${WEBOS_DB8_BACKEND})
src/storage-sandwich/MojDbSandwichEnv.cpp
src/storage-sandwich/MojDbSandwichIndex.cpp
src/storage-sandwich/MojDbSandwichItem.cpp
src/storage-sandwich/MojDbSandwichLazyUpdater.cpp
)

set (DB_BACKEND_WRAPPER_CFLAGS "${DB_BACKEND_WRAPPER_CFLAGS} -I${CMAKE_SOURCE_DIR}/src/storage-sandwich -DMOJ_USE_SANDWICH")
Expand Down
4 changes: 3 additions & 1 deletion src/db/MojDb.cpp
Expand Up @@ -1353,6 +1353,8 @@ MojErr MojDb::commitBatch(MojDbReq& req)
{
LOG_TRACE("Entering function %s", __FUNCTION__);

bool schemaLocked = req.schemaLocked();

// commit current batch and get things reset for next batch
// Can NOT have db cursor open at this stage and new queries have to be started
// Use with Caution otherwise you get db fatal errors
Expand All @@ -1374,7 +1376,7 @@ MojErr MojDb::commitBatch(MojDbReq& req)

req.beginBatch();

err = beginReq(req, false);
err = beginReq(req, schemaLocked);
if (err != MojErrNone)
LOG_DEBUG("[db_mojodb] CommitBatch ended: err= %d\n", (int)err);

Expand Down
21 changes: 20 additions & 1 deletion src/storage-sandwich/MojDbSandwichDatabase.cpp
Expand Up @@ -24,6 +24,8 @@
#include "db/MojDb.h"
#include "defs.h"

#include "MojDbSandwichLazyUpdater.h"

////////////////////MojDbSandwichDatabase////////////////////////////////////////////

MojDbSandwichDatabase::MojDbSandwichDatabase(const MojDbSandwichEngine::BackendDb::Part& part)
Expand All @@ -47,16 +49,29 @@ MojErr MojDbSandwichDatabase::open(const MojChar* dbName, MojDbSandwichEngine* e
MojErr err = m_name.assign(dbName);
MojErrCheck(err);

if (engine()->lazySync())
engine()->getUpdater()->open( getDb() );

return MojErrNone;
}

leveldb::DB* MojDbSandwichDatabase::getDb()
{
MojDbSandwichEngine::BackendDb& backendDb = engine()->impl();
leveldb::DB* db = (*backendDb).get();

return db;
}

MojErr MojDbSandwichDatabase::close()
{
LOG_TRACE("Entering function %s", __FUNCTION__);

MojErr err = MojErrNone;
if (m_db.Valid()) {
err = closeImpl();
if (engine()->lazySync())
engine()->getUpdater()->close( getDb() );
}
return err;
}
Expand Down Expand Up @@ -271,7 +286,7 @@ MojErr MojDbSandwichDatabase::beginTxn(MojRefCountedPtr<MojDbStorageTxn>& txnOut
{
LOG_TRACE("Entering function %s", __FUNCTION__);
//MojAssert( m_db );
MojRefCountedPtr<MojDbSandwichEnvTxn> txn(new MojDbSandwichEnvTxn(m_engine->impl()));
MojRefCountedPtr<MojDbSandwichEnvTxn> txn(new MojDbSandwichEnvTxn(m_engine->impl(), *engine()));
MojAllocCheck(txn.get());

txnOut = txn;
Expand Down Expand Up @@ -411,5 +426,9 @@ void MojDbSandwichDatabase::postUpdate(MojDbStorageTxn* txn, MojSize size)
if (txn) {
// TODO: implement quotas
// XXX: static_cast<MojDbSandwichTxn*>(txn)->didUpdate(size);
} else {
if (engine()->lazySync())
engine()->getUpdater()->sendEvent( getDb() );
}
}

2 changes: 2 additions & 0 deletions src/storage-sandwich/MojDbSandwichDatabase.h
Expand Up @@ -61,6 +61,8 @@ class MojDbSandwichDatabase final : public MojDbStorageDatabase
MojDbSandwichEngine::BackendDb::Part& impl() { return m_db; }
MojDbSandwichEngine* engine() { return m_engine; }

leveldb::DB* getDb();

private:
friend class MojDbSandwichEngine;
friend class MojDbSandwichIndex;
Expand Down
36 changes: 33 additions & 3 deletions src/storage-sandwich/MojDbSandwichEngine.cpp
Expand Up @@ -36,6 +36,7 @@
#include "core/MojObjectSerialization.h"
#include "core/MojString.h"
#include "core/MojTokenSet.h"
#include "MojDbSandwichLazyUpdater.h"

#include <sys/statvfs.h>

Expand All @@ -53,23 +54,43 @@ leveldb::Options MojDbSandwichEngine::OpenOptions;
////////////////////MojDbSandwichEngine////////////////////////////////////////////

MojDbSandwichEngine::MojDbSandwichEngine()
: m_isOpen(false)
: m_isOpen(false), m_lazySync(false), m_updater(NULL)
{
m_updater = new MojDbSandwichLazyUpdater;
}


MojDbSandwichEngine::~MojDbSandwichEngine()
{
MojErr err = close();
MojErrCatchAll(err);

if (m_updater)
delete m_updater;
}

MojErr MojDbSandwichEngine::configure(const MojObject& config)
{
LOG_TRACE("Entering function %s", __FUNCTION__);

if (!config.get("sync", WriteOptions.sync)) {
bool found=false;
MojInt32 syncOption=0;
MojErr err = config.get("sync", syncOption, found);
MojErrCheck(err);
if (false == found) {
WriteOptions.sync = true;
} else {
switch(syncOption) {
case 2:
WriteOptions.sync = false;
m_lazySync = true;
break;

case 1:
case 0:
WriteOptions.sync = !!syncOption;
break;
}
}

if (!config.get("fill_cache", ReadOptions.fill_cache)) {
Expand Down Expand Up @@ -125,6 +146,9 @@ MojErr MojDbSandwichEngine::open(const MojChar* path)
err = m_path.assign(path);
MojErrCheck(err);

if (lazySync())
m_updater->start();

return MojErrNone;
}

Expand Down Expand Up @@ -173,6 +197,9 @@ MojErr MojDbSandwichEngine::open(const MojChar* path, MojDbEnv* env)
MojErrCheck(err);
m_isOpen = true;

if (lazySync())
m_updater->start();

return MojErrNone;
}

Expand All @@ -199,6 +226,9 @@ MojErr MojDbSandwichEngine::close()
m_env.reset();
m_isOpen = false;

if (lazySync())
m_updater->stop();

return err;
}

Expand All @@ -207,7 +237,7 @@ MojErr MojDbSandwichEngine::beginTxn(MojRefCountedPtr<MojDbStorageTxn>& txnOut)
LOG_TRACE("Entering function %s", __FUNCTION__);
MojAssert(!txnOut.get());

MojRefCountedPtr<MojDbSandwichEnvTxn> txn(new MojDbSandwichEnvTxn(m_db));
MojRefCountedPtr<MojDbSandwichEnvTxn> txn(new MojDbSandwichEnvTxn(m_db, *this));
MojAllocCheck(txn.get());
txnOut = txn;

Expand Down
8 changes: 8 additions & 0 deletions src/storage-sandwich/MojDbSandwichEngine.h
Expand Up @@ -31,6 +31,7 @@
class MojDbSandwichDatabase;
class MojDbSandwichEnv;
class MojDbSandwichSeq;
class MojDbSandwichLazyUpdater;

class MojDbSandwichEngine : public MojDbStorageEngine
{
Expand Down Expand Up @@ -60,6 +61,10 @@ class MojDbSandwichEngine : public MojDbStorageEngine
static const leveldb::WriteOptions& getWriteOptions() { return WriteOptions; }
static const leveldb::ReadOptions& getReadOptions() { return ReadOptions; }
static const leveldb::Options& getOpenOptions() { return OpenOptions; }

MojDbSandwichLazyUpdater* getUpdater() const { return m_updater; }
bool lazySync() const { return m_lazySync; }

private:
typedef MojVector<MojRefCountedPtr<MojDbSandwichDatabase> > DatabaseVec;
typedef MojVector<MojRefCountedPtr<MojDbSandwichSeq> > SequenceVec;
Expand All @@ -76,6 +81,9 @@ class MojDbSandwichEngine : public MojDbStorageEngine
static leveldb::ReadOptions ReadOptions;
static leveldb::WriteOptions WriteOptions;
static leveldb::Options OpenOptions;

bool m_lazySync;
MojDbSandwichLazyUpdater* m_updater;
};

#endif /* MOJDBLEVELENGINE_H_ */
156 changes: 156 additions & 0 deletions src/storage-sandwich/MojDbSandwichLazyUpdater.cpp
@@ -0,0 +1,156 @@
/* @@@LICENSE
*
* Copyright (c) 2014 LG Electronics, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* LICENSE@@@ */

#include "db/MojDb.h"
#include "MojDbSandwichLazyUpdater.h"
#include <time.h>

////////////////////MojDbSandwichLazyUpdater////////////////////////////////////////////

MojDbSandwichLazyUpdater::MojDbSandwichLazyUpdater()
: m_thread(MojInvalidThread), m_stop(false)
{
}

MojDbSandwichLazyUpdater::~MojDbSandwichLazyUpdater()
{
deinit();
}

MojErr MojDbSandwichLazyUpdater::start()
{
MojErr err = MojErrNone;

if (MojInvalidThread == m_thread)
{
err = MojThreadCreate(m_thread, &threadMain, this);
MojErrCheck(err);
}
m_stop = false;

return (err);
}

MojErr MojDbSandwichLazyUpdater::stop()
{
MojErr err = MojErrNone;

m_stop = true;
return (err);
}

MojErr MojDbSandwichLazyUpdater::deinit()
{
MojErr err = MojErrNone;

stop();
if (MojInvalidThread != m_thread)
{
MojErr threadErr = MojErrNone;
MojErr joinErr = MojThreadJoin(m_thread, threadErr);

MojErrAccumulate(err, threadErr);
MojErrAccumulate(err, joinErr);
}

return (err);
}

MojErr MojDbSandwichLazyUpdater::threadMain(void* arg)
{
MojDbSandwichLazyUpdater* thiz_class = (MojDbSandwichLazyUpdater*) arg;
MojAssert(thiz_class);

while (thiz_class->m_stop == false) {
thiz_class->sync();
MojSleep(UpdaterIntervalMsec * 1000);
}

return MojErrNone;
}

MojErr MojDbSandwichLazyUpdater::open(leveldb::DB* pdb)
{
if(pdb == 0)
return MojErrUnknown;

MojThreadGuard guard(m_mutex);
if(m_dbs.find(pdb) == m_dbs.end()) {
m_dbs[pdb] = false;
}

return MojErrNone;
}

MojErr MojDbSandwichLazyUpdater::close(leveldb::DB* pdb)
{
MojThreadGuard guard(m_mutex);
if(m_dbs.find(pdb) != m_dbs.end()) {
onesync(pdb);
m_dbs.erase(pdb);
}

return MojErrNone;
}

MojErr MojDbSandwichLazyUpdater::sendEvent(leveldb::DB* pdb)
{
MojThreadGuard guard(m_mutex);
m_dbs[pdb] = true;

return MojErrNone;
}

MojErr MojDbSandwichLazyUpdater::sync()
{
Container dbs_copy;

MojThreadGuard guard(m_mutex);
dbs_copy.clear();
for (Container::iterator it = m_dbs.begin();
it != m_dbs.end();
++it)
{
if(it->first && it->second == true) {
it->second = false;
dbs_copy[it->first] = true;
}
}
guard.unlock();

for (Container::iterator it = dbs_copy.begin();
it != dbs_copy.end();
++it) {
if(it->first && it->second == true) {
onesync(it->first);
}
}

LOG_DEBUG("[db_ldb] do sync\n");
return MojErrNone;
}

void MojDbSandwichLazyUpdater::onesync(leveldb::DB* pdb)
{
// Delete meaning less data for sync.
// TODO: Rationalize followings.
leveldb::WriteOptions write_options;
write_options.sync = true;
pdb->Delete(write_options, "_______dummy______");
}

0 comments on commit 1a90b24

Please sign in to comment.