Skip to content

Commit

Permalink
Merge pull request #2225 from MonsieurNicolas/squashDBschema10
Browse files Browse the repository at this point in the history 
Squash db schema version 10

Reviewed-by: marta-lokhova
  • Loading branch information
@latobarita
latobarita committed Aug 15, 2019
2 parents 512dfa6 + b36409a commit c5fb1e3
Show file tree
Hide file tree
Showing 9 changed files with 35 additions and 462 deletions.
52 changes: 13 additions & 39 deletions src/database/Database.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ using namespace std;

bool Database::gDriversRegistered = false;

// smallest schema version supported
static unsigned long const MIN_SCHEMA_VERSION = 9;
static unsigned long const SCHEMA_VERSION = 10;

// These should always match our compiled version precisely, since we are
Expand Down Expand Up @@ -189,49 +191,12 @@ Database::applySchemaUpgrade(unsigned long vers)
soci::transaction tx(mSession);
switch (vers)
{
case 7:
Upgrades::dropAll(*this);
mSession << "ALTER TABLE accounts ADD buyingliabilities BIGINT "
"CHECK (buyingliabilities >= 0)";
mSession << "ALTER TABLE accounts ADD sellingliabilities BIGINT "
"CHECK (sellingliabilities >= 0)";
mSession << "ALTER TABLE trustlines ADD buyingliabilities BIGINT "
"CHECK (buyingliabilities >= 0)";
mSession << "ALTER TABLE trustlines ADD sellingliabilities BIGINT "
"CHECK (sellingliabilities >= 0)";
break;

case 8:
mSession << "ALTER TABLE peers RENAME flags TO type";
mSession << "UPDATE peers SET type = 2*type";
break;
case 9:
// Update schema for signers
mSession << "ALTER TABLE accounts ADD signers TEXT";
mApp.getLedgerTxnRoot().writeSignersTableIntoAccountsTable();
mSession << "DROP TABLE IF EXISTS signers";

// Update schema for base-64 encoding
mApp.getLedgerTxnRoot().encodeDataNamesBase64();
mApp.getLedgerTxnRoot().encodeHomeDomainsBase64();

// Update schema for simplified offers table
mApp.getLedgerTxnRoot().writeOffersIntoSimplifiedOffersTable();
break;
case 10:
// add tracking table information
mApp.getHerderPersistence().createQuorumTrackingTable(mSession);
break;
default:
if (vers <= 6)
{
throw std::runtime_error(
"Database version is too old, must use at least 7");
}
else
{
throw std::runtime_error("Unknown DB schema version");
}
throw std::runtime_error("Unknown DB schema version");
}
tx.commit();
}
Expand All @@ -240,6 +205,14 @@ void
Database::upgradeToCurrentSchema()
{
auto vers = getDBSchemaVersion();
if (vers < MIN_SCHEMA_VERSION)
{
std::string s = ("DB schema version " + std::to_string(vers) +
" is older than minimum supported schema " +
std::to_string(MIN_SCHEMA_VERSION));
throw std::runtime_error(s);
}

if (vers > SCHEMA_VERSION)
{
std::string s = ("DB schema version " + std::to_string(vers) +
Expand Down Expand Up @@ -390,6 +363,7 @@ Database::initialize()

// only time this section should be modified is when
// consolidating changes found in applySchemaUpgrade here
Upgrades::dropAll(*this);
mApp.getLedgerTxnRoot().dropAccounts();
mApp.getLedgerTxnRoot().dropOffers();
mApp.getLedgerTxnRoot().dropTrustLines();
Expand All @@ -402,7 +376,7 @@ Database::initialize()
HerderPersistence::dropAll(*this);
mApp.getLedgerTxnRoot().dropData();
BanManager::dropAll(*this);
putSchemaVersion(6);
putSchemaVersion(MIN_SCHEMA_VERSION);

LOG(INFO) << "* ";
LOG(INFO) << "* The database has been initialized";
Expand Down
24 changes: 0 additions & 24 deletions src/ledger/LedgerTxn.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2008,28 +2008,4 @@ LedgerTxnRoot::Impl::getFromBestOffersCache(
throw;
}
}

void
LedgerTxnRoot::writeSignersTableIntoAccountsTable()
{
mImpl->writeSignersTableIntoAccountsTable();
}

void
LedgerTxnRoot::encodeDataNamesBase64()
{
mImpl->encodeDataNamesBase64();
}

void
LedgerTxnRoot::encodeHomeDomainsBase64()
{
mImpl->encodeHomeDomainsBase64();
}

void
LedgerTxnRoot::writeOffersIntoSimplifiedOffersTable()
{
mImpl->writeOffersIntoSimplifiedOffersTable();
}
}
5 changes: 0 additions & 5 deletions src/ledger/LedgerTxn.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -597,11 +597,6 @@ class LedgerTxnRoot : public AbstractLedgerTxnParent

void rollbackChild() override;

void writeSignersTableIntoAccountsTable();
void encodeDataNamesBase64();
void encodeHomeDomainsBase64();

void writeOffersIntoSimplifiedOffersTable();
uint32_t prefetch(std::unordered_set<LedgerKey> const& keys);
double getPrefetchHitRate() const;
};
Expand Down
175 changes: 13 additions & 162 deletions src/ledger/LedgerTxnAccountSQL.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,74 +128,6 @@ LedgerTxnRoot::Impl::loadInflationWinners(size_t maxWinners,
return winners;
}

void
LedgerTxnRoot::Impl::writeSignersTableIntoAccountsTable()
{
throwIfChild();
soci::transaction sqlTx(mDatabase.getSession());

CLOG(INFO, "Ledger") << "Loading all signers from signers table";
std::map<std::string, xdr::xvector<Signer, 20>> signersByAccount;

{
std::string accountIDStrKey, pubKey;
Signer signer;

auto prep = mDatabase.getPreparedStatement(
"SELECT accountid, publickey, weight FROM signers");
auto& st = prep.statement();
st.exchange(soci::into(accountIDStrKey));
st.exchange(soci::into(pubKey));
st.exchange(soci::into(signer.weight));
st.define_and_bind();
{
auto timer = mDatabase.getSelectTimer("signer");
st.execute(true);
}
while (st.got_data())
{
signer.key = KeyUtils::fromStrKey<SignerKey>(pubKey);
signersByAccount[accountIDStrKey].emplace_back(signer);
st.fetch();
}
}

size_t numAccountsUpdated = 0;
for (auto& kv : signersByAccount)
{
std::sort(kv.second.begin(), kv.second.end(),
[](Signer const& lhs, Signer const& rhs) {
return lhs.key < rhs.key;
});
std::string signers(decoder::encode_b64(xdr::xdr_to_opaque(kv.second)));

auto prep = mDatabase.getPreparedStatement(
"UPDATE accounts SET signers = :v1 WHERE accountID = :id");
auto& st = prep.statement();
st.exchange(soci::use(signers, "v1"));
st.exchange(soci::use(kv.first, "id"));
st.define_and_bind();
st.execute(true);
if (static_cast<size_t>(st.get_affected_rows()) != 1)
{
throw std::runtime_error("Could not update data in SQL");
}

if ((++numAccountsUpdated & 0xfff) == 0xfff ||
(numAccountsUpdated == signersByAccount.size()))
{
CLOG(INFO, "Ledger")
<< "Wrote signers for " << numAccountsUpdated << " accounts";
}
}

sqlTx.commit();

// Clearing the cache does not throw
mEntryCache.clear();
mBestOffersCache.clear();
}

class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation<void>
{
Database& mDB;
Expand Down Expand Up @@ -530,106 +462,25 @@ LedgerTxnRoot::Impl::dropAccounts()
mDatabase.getSession()
<< "CREATE TABLE accounts"
"("
"accountid VARCHAR(56) PRIMARY KEY,"
"balance BIGINT NOT NULL CHECK (balance >= 0),"
"seqnum BIGINT NOT NULL,"
"numsubentries INT NOT NULL CHECK (numsubentries >= 0),"
"inflationdest VARCHAR(56),"
"homedomain VARCHAR(32) NOT NULL,"
"thresholds TEXT NOT NULL,"
"flags INT NOT NULL,"
"lastmodified INT NOT NULL"
"accountid VARCHAR(56) PRIMARY KEY,"
"balance BIGINT NOT NULL CHECK (balance >= 0),"
"buyingliabilities BIGINT CHECK (buyingliabilities >= 0),"
"sellingliabilities BIGINT CHECK (sellingliabilities >= 0),"
"seqnum BIGINT NOT NULL,"
"numsubentries INT NOT NULL CHECK (numsubentries >= "
"0),"
"inflationdest VARCHAR(56),"
"homedomain VARCHAR(32) NOT NULL,"
"thresholds TEXT NOT NULL,"
"flags INT NOT NULL,"
"signers TEXT,"
"lastmodified INT NOT NULL"
");";
mDatabase.getSession() << "CREATE TABLE signers"
"("
"accountid VARCHAR(56) NOT NULL,"
"publickey VARCHAR(56) NOT NULL,"
"weight INT NOT NULL,"
"PRIMARY KEY (accountid, publickey)"
");";
mDatabase.getSession()
<< "CREATE INDEX signersaccount ON signers (accountid)";
mDatabase.getSession()
<< "CREATE INDEX accountbalances ON accounts (balance) WHERE "
"balance >= 1000000000";
}

static std::vector<std::pair<std::string, std::string>>
loadHomeDomainsToEncode(Database& db)
{
std::string accountID, homeDomain;

std::string sql = "SELECT accountid, homedomain FROM accounts "
"WHERE length(homedomain) > 0";

auto prep = db.getPreparedStatement(sql);
auto& st = prep.statement();
st.exchange(soci::into(accountID));
st.exchange(soci::into(homeDomain));
st.define_and_bind();
st.execute(true);

std::vector<std::pair<std::string, std::string>> res;
while (st.got_data())
{
res.emplace_back(accountID, homeDomain);
st.fetch();
}
return res;
}

static void
writeEncodedHomeDomain(Database& db, std::string const& accountID,
std::string const& homeDomain)
{
std::string encodedHomeDomain = decoder::encode_b64(homeDomain);

std::string sql =
"UPDATE accounts SET homedomain = :v1 WHERE accountid = :v2";

auto prep = db.getPreparedStatement(sql);
auto& st = prep.statement();
st.exchange(soci::use(encodedHomeDomain));
st.exchange(soci::use(accountID));
st.define_and_bind();
st.execute(true);
if (static_cast<size_t>(st.get_affected_rows()) != 1)
{
throw std::runtime_error("could not update SQL");
}
}

void
LedgerTxnRoot::Impl::encodeHomeDomainsBase64()
{
throwIfChild();
mEntryCache.clear();
mBestOffersCache.clear();

CLOG(INFO, "Ledger") << "Loading all home domains from accounts table";
auto homeDomainsToEncode = loadHomeDomainsToEncode(mDatabase);

if (!mDatabase.isSqlite())
{
auto& session = mDatabase.getSession();
session << "ALTER TABLE accounts ALTER COLUMN homedomain "
"SET DATA TYPE VARCHAR(44)";
}

size_t numUpdated = 0;
for (auto const& kv : homeDomainsToEncode)
{
writeEncodedHomeDomain(mDatabase, kv.first, kv.second);

if ((++numUpdated & 0xfff) == 0xfff ||
(numUpdated == homeDomainsToEncode.size()))
{
CLOG(INFO, "Ledger")
<< "Wrote home domains for " << numUpdated << " accounts";
}
}
}

class BulkLoadAccountsOperation
: public DatabaseTypeSpecificOperation<std::vector<LedgerEntry>>
{
Expand Down
65 changes: 0 additions & 65 deletions src/ledger/LedgerTxnDataSQL.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,71 +289,6 @@ LedgerTxnRoot::Impl::dropData()
");";
}

static std::vector<LedgerEntry>
loadDataToEncode(Database& db)
{
std::string accountID, dataName, dataValue;
uint32_t lastModified;

std::string sql = "SELECT accountid, dataname, datavalue, lastmodified "
"FROM accountdata";

auto prep = db.getPreparedStatement(sql);
auto& st = prep.statement();
st.exchange(soci::into(accountID));
st.exchange(soci::into(dataName));
st.exchange(soci::into(dataValue));
st.exchange(soci::into(lastModified));
st.define_and_bind();
st.execute(true);

std::vector<LedgerEntry> res;
while (st.got_data())
{
res.emplace_back();
auto& le = res.back();
le.data.type(DATA);

auto& de = le.data.data();
de.accountID = KeyUtils::fromStrKey<PublicKey>(accountID);
de.dataName = dataName;
decoder::decode_b64(dataValue, de.dataValue);
le.lastModifiedLedgerSeq = lastModified;

st.fetch();
}
return res;
}

void
LedgerTxnRoot::Impl::encodeDataNamesBase64()
{
throwIfChild();
mEntryCache.clear();
mBestOffersCache.clear();

CLOG(INFO, "Ledger")
<< "Loading all data entries from the accountdata table";
auto dataToEncode = loadDataToEncode(mDatabase);

// Note: The table must be recreated since dataname is part of the primary
// key, so there could be a collision when updating it
dropData();
if (!mDatabase.isSqlite())
{
auto& session = mDatabase.getSession();
session << "ALTER TABLE accountdata ALTER COLUMN dataname "
"SET DATA TYPE VARCHAR(88)";
}
if (!dataToEncode.empty())
{
BulkUpsertDataOperation op(mDatabase, dataToEncode);
mDatabase.doDatabaseTypeSpecificOperation(op);
CLOG(INFO, "Ledger")
<< "Wrote " << dataToEncode.size() << " data entries";
}
}

class BulkLoadDataOperation
: public DatabaseTypeSpecificOperation<std::vector<LedgerEntry>>
{
Expand Down

0 comments on commit c5fb1e3

Please sign in to comment.