Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimize SQL Statements / result gathering #2007

Merged
merged 3 commits into from
Oct 7, 2021
Merged

Optimize SQL Statements / result gathering #2007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f3e1d94
Optimize gathering of query results
matthiasstraka Oct 6, 2021
b4338de
SQL statement simplifications
matthiasstraka Oct 6, 2021
c997f81
Move collected metadata/resources to CdsObject
matthiasstraka Oct 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/cds_objects.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,12 @@ class CdsObject {
this->metaData = metaData;
}

/// \brief Set entire metadata dictionary by moving
void setMetaData(std::vector<std::pair<std::string, std::string>>&& metaData)
{
this->metaData = std::move(metaData);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know if it's a good idea to have setters moving stuff.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not? Isn't this a very common pattern and one of the main reasons for move?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm nvm.

}

/// \brief Add a single metadata value.
void addMetaData(const metadata_fields_t key, const std::string& value)
{
Expand Down Expand Up @@ -330,6 +336,12 @@ class CdsObject {
resources = res;
}

/// \brief Set resources by move
void setResources(std::vector<std::shared_ptr<CdsResource>>&& res)
{
resources = std::move(res);
}

/// \brief Search resources for given handler id
bool hasResource(int id) const
{
Expand Down
70 changes: 42 additions & 28 deletions src/database/sql_database.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1098,15 +1098,15 @@ std::map<int, int> SQLDatabase::getChildCounts(const std::vector<int>& contId, b
}

beginTransaction("getChildCounts");
auto res = select(fmt::format("SELECT {0}, COUNT(*) FROM {1} WHERE {2} GROUP BY {0}",
auto res = select(fmt::format("SELECT {0}, COUNT(*) FROM {1} WHERE {2} GROUP BY {0} ORDER BY {0}",
identifier("parent_id"), identifier(CDS_OBJECT_TABLE), fmt::join(where, " AND ")));
commit("getChildCounts");

std::map<int, int> result;
if (res) {
std::unique_ptr<SQLRow> row;
while ((row = res->nextRow())) {
result.emplace(row->col_int(0, INVALID_OBJECT_ID), row->col_int(1, 0));
result.emplace_hint(result.end(), row->col_int(0, INVALID_OBJECT_ID), row->col_int(1, 0));
}
}
return result;
Expand Down Expand Up @@ -1380,26 +1380,29 @@ std::shared_ptr<CdsObject> SQLDatabase::createObjectFromRow(const std::unique_pt

auto metaData = retrieveMetaDataForObject(obj->getID());
if (!metaData.empty()) {
obj->setMetaData(metaData);
obj->setMetaData(std::move(metaData));
} else if (obj->getRefID() != CDS_ID_ROOT) {
metaData = retrieveMetaDataForObject(obj->getRefID());
if (!metaData.empty())
obj->setMetaData(metaData);
obj->setMetaData(std::move(metaData));
}

std::string auxdataStr = fallbackString(getCol(row, BrowseCol::Auxdata), getCol(row, BrowseCol::RefAuxdata));
std::map<std::string, std::string> aux = dictDecode(auxdataStr);
obj->setAuxData(aux);

bool resourceZeroOk = false;
auto resources = retrieveResourcesForObject(obj->getID());
if (!resources.empty()) {
obj->setResources(resources);
resourceZeroOk = true;
obj->setResources(std::move(resources));
} else if (obj->getRefID() != CDS_ID_ROOT) {
resources = retrieveResourcesForObject(obj->getRefID());
if (!resources.empty())
obj->setResources(resources);
if (!resources.empty()) {
resourceZeroOk = true;
obj->setResources(std::move(resources));
}
}
auto resourceZeroOk = !resources.empty();

obj->setVirtual((obj->getRefID() && obj->isPureItem()) || (obj->isItem() && !obj->isPureItem())); // gets set to true for virtual containers below

Expand Down Expand Up @@ -1475,19 +1478,21 @@ std::shared_ptr<CdsObject> SQLDatabase::createObjectFromSearchRow(const std::uni

auto metaData = retrieveMetaDataForObject(obj->getID());
if (!metaData.empty())
obj->setMetaData(metaData);
obj->setMetaData(std::move(metaData));

bool resourceZeroOk = false;
auto resources = retrieveResourcesForObject(obj->getID());
if (!resources.empty()) {
obj->setResources(resources);
resourceZeroOk = true;
obj->setResources(std::move(resources));
} else if (obj->getRefID() != CDS_ID_ROOT) {
resources = retrieveResourcesForObject(obj->getRefID());
if (!resources.empty())
obj->setResources(resources);
if (!resources.empty()) {
resourceZeroOk = true;
obj->setResources(std::move(resources));
}
}

auto resourceZeroOk = !resources.empty();

if (obj->isItem()) {
if (!resourceZeroOk)
throw_std_runtime_error("tried to create object without at least one resource");
Expand Down Expand Up @@ -1518,6 +1523,8 @@ std::vector<std::pair<std::string, std::string>> SQLDatabase::retrieveMetaDataFo
return {};

std::vector<std::pair<std::string, std::string>> metaData;
metaData.reserve(res->getNumRows());

std::unique_ptr<SQLRow> row;
while ((row = res->nextRow())) {
metaData.emplace_back(getCol(row, MetadataCol::PropertyName), getCol(row, MetadataCol::PropertyValue));
Expand Down Expand Up @@ -1582,9 +1589,13 @@ std::string SQLDatabase::incrementUpdateIDs(const std::unordered_set<int>& ids)

std::unordered_set<int> SQLDatabase::getObjects(int parentID, bool withoutContainer)
{
auto colId = identifier("id");
auto table = identifier(CDS_OBJECT_TABLE);
auto colParentId = identifier("parent_id");

auto getSql = (withoutContainer) //
? fmt::format("SELECT {0}id{1} FROM {0}{2}{1} WHERE {0}parent_id{1} = {3} AND {0}object_type{1} != {4}", table_quote_begin, table_quote_end, CDS_OBJECT_TABLE, parentID, OBJECT_TYPE_CONTAINER)
: fmt::format("SELECT {0}id{1} FROM {0}{2}{1} WHERE {0}parent_id{1} = {3}", table_quote_begin, table_quote_end, CDS_OBJECT_TABLE, parentID);
? fmt::format("SELECT {} FROM {} WHERE {} = {} AND {} != {}", colId, table, colParentId, parentID, identifier("object_type"), OBJECT_TYPE_CONTAINER)
: fmt::format("SELECT {} FROM {} WHERE {} = {}", colId, table, colParentId, parentID);
auto res = select(getSql);
if (!res)
throw_std_runtime_error("db error");
Expand All @@ -1611,7 +1622,8 @@ std::unique_ptr<Database::ChangedContainers> SQLDatabase::removeObjects(const st
throw_std_runtime_error("Tried to delete a forbidden ID ({})", *it);
}

auto res = select(fmt::format("SELECT {0}id{1}, {0}object_type{1} FROM {0}{2}{1} WHERE {0}id{1} IN ({3})", table_quote_begin, table_quote_end, CDS_OBJECT_TABLE, fmt::join(list, ",")));
auto res = select(fmt::format("SELECT {0}, {1} FROM {2} WHERE {0} IN ({3})",
identifier("id"), identifier("object_type"), identifier(CDS_OBJECT_TABLE), fmt::join(list, ",")));
if (!res)
throw_std_runtime_error("sql error");

Expand Down Expand Up @@ -1673,7 +1685,8 @@ void SQLDatabase::_removeObjects(const std::vector<std::int32_t>& objectIDs)

std::unique_ptr<Database::ChangedContainers> SQLDatabase::removeObject(int objectID, bool all)
{
auto res = select(fmt::format("SELECT {0}object_type{1}, {0}ref_id{1} FROM {0}{2}{1} WHERE {0}id{1} = {3} LIMIT 1", table_quote_begin, table_quote_end, CDS_OBJECT_TABLE, objectID));
auto res = select(fmt::format("SELECT {}, {} FROM {} WHERE {} = {} LIMIT 1",
identifier("object_type"), identifier("ref_id"), identifier(CDS_OBJECT_TABLE), identifier("id"), objectID));
if (!res)
return nullptr;

Expand Down Expand Up @@ -1822,16 +1835,17 @@ std::unique_ptr<Database::ChangedContainers> SQLDatabase::_purgeEmptyContainers(
if (maybeEmpty->upnp.empty() && maybeEmpty->ui.empty())
return {};

constexpr auto tabAlias = "fol";
constexpr auto childAlias = "cld";
auto tabAlias = identifier("fol");
auto childAlias = identifier("cld");
auto colId = identifier("id");
auto fields = std::vector {
fmt::format("{}.{}", identifier(tabAlias), identifier("id")),
fmt::format("COUNT({}.{})", identifier(childAlias), identifier("parent_id")),
fmt::format("{}.{}", identifier(tabAlias), identifier("parent_id")),
fmt::format("{}.{}", identifier(tabAlias), identifier("flags")),
fmt::format("{}.{}", tabAlias, colId),
fmt::format("COUNT({}.{})", childAlias, identifier("parent_id")),
fmt::format("{}.{}", tabAlias, identifier("parent_id")),
fmt::format("{}.{}", tabAlias, identifier("flags")),
};
std::string selectSql = fmt::format("SELECT {2} FROM {5} {3} LEFT JOIN {5} {4} ON {0}{3}{1}.{0}id{1} = {0}{4}{1}.{0}parent_id{1} WHERE {0}{3}{1}.{0}object_type{1} = {6} AND {0}{3}{1}.{0}id{1} ",
table_quote_begin, table_quote_end, fmt::join(fields, ","), tabAlias, childAlias, CDS_OBJECT_TABLE, quote(OBJECT_TYPE_CONTAINER));
std::string selectSql = fmt::format("SELECT {2} FROM {5} {3} LEFT JOIN {5} {4} ON {3}.{0}id{1} = {4}.{0}parent_id{1} WHERE {3}.{0}object_type{1} = {6} AND {3}.{0}id{1}",
table_quote_begin, table_quote_end, fmt::join(fields, ","), tabAlias, childAlias, identifier(CDS_OBJECT_TABLE), OBJECT_TYPE_CONTAINER);

std::vector<std::int32_t> del;

Expand All @@ -1848,7 +1862,7 @@ std::unique_ptr<Database::ChangedContainers> SQLDatabase::_purgeEmptyContainers(
again = false;

if (!selUpnp.empty()) {
auto sql = fmt::format("{2} IN ({3}) GROUP BY {0}{4}{1}.{0}id{1}", table_quote_begin, table_quote_end, selectSql, fmt::join(selUpnp, ","), tabAlias);
auto sql = fmt::format("{} IN ({}) GROUP BY {}.{}", selectSql, fmt::join(selUpnp, ","), tabAlias, colId);
log_debug("upnp-sql: {}", sql);
std::shared_ptr<SQLResult> res = select(sql);
selUpnp.clear();
Expand All @@ -1868,7 +1882,7 @@ std::unique_ptr<Database::ChangedContainers> SQLDatabase::_purgeEmptyContainers(
}

if (!selUi.empty()) {
auto sql = fmt::format("{2} IN ({3}) GROUP BY {0}{4}{1}.{0}id{1}", table_quote_begin, table_quote_end, selectSql, fmt::join(selUi, ","), tabAlias);
auto sql = fmt::format("{} IN ({}) GROUP BY {}.{}", selectSql, fmt::join(selUi, ","), tabAlias, colId);
log_debug("ui-sql: {}", sql);
std::shared_ptr<SQLResult> res = select(sql);
selUi.clear();
Expand Down