Skip to content

Commit

Permalink
Merge "Merge commit 'couchbase/neo' into 'couchbase/master'"
Browse files Browse the repository at this point in the history
  • Loading branch information
Gerrit Code Review committed Jan 3, 2023
2 parents e5eb5c5 + 2bbce11 commit 759684a
Show file tree
Hide file tree
Showing 23 changed files with 336 additions and 166 deletions.
9 changes: 2 additions & 7 deletions engines/ep/src/checkpoint.cc
View file
Expand Up @@ -386,13 +386,8 @@ QueueDirtyResult Checkpoint::queueDirty(const queued_item& qi) {

bool Checkpoint::canDedup(const queued_item& existing,
const queued_item& in) const {
auto isDurabilityOp = [](const queued_item& qi_) -> bool {
const auto op = qi_->getOperation();
return op == queue_op::pending_sync_write ||
op == queue_op::commit_sync_write ||
op == queue_op::abort_sync_write;
};
return !(isDurabilityOp(existing) || isDurabilityOp(in));
return !(existing->isAnySyncWriteOp() || in->isAnySyncWriteOp() ||
!in->canDeduplicate());
}

uint64_t Checkpoint::getMinimumCursorSeqno() const {
Expand Down
21 changes: 14 additions & 7 deletions engines/ep/src/collections/collections_types.cc
View file
Expand Up @@ -12,6 +12,7 @@
#include "collections/collections_types.h"
#include "systemevent_factory.h"

#include <fmt/ostream.h>
#include <mcbp/protocol/unsigned_leb128.h>
#include <memcached/dockey.h>
#include <nlohmann/json.hpp>
Expand Down Expand Up @@ -222,14 +223,20 @@ std::ostream& operator<<(std::ostream& os, const ScopeSharedMetaData& meta) {

} // namespace VB

std::ostream& operator<<(std::ostream& os, const CollectionMetaData& meta) {
os << "sid:" << meta.sid << ",cid:" << meta.cid << ",name:" << meta.name;

if (meta.maxTtl) {
os << ",maxTTl:" << meta.maxTtl->count();
}
std::string to_string(const CollectionMetaData& collection) {
return fmt::format(
"cid:{}, name:{}, ttl:{{{}, {}}}, sid:{}, {}, {}",
collection.cid.to_string(),
collection.name,
collection.maxTtl.has_value(),
collection.maxTtl.value_or(std::chrono::seconds(0)).count(),
collection.sid.to_string(),
collection.metered,
collection.canDeduplicate);
}

return os;
std::ostream& operator<<(std::ostream& os, const CollectionMetaData& meta) {
return os << to_string(meta);
}

std::ostream& operator<<(std::ostream& os, const ScopeMetaData& meta) {
Expand Down
37 changes: 30 additions & 7 deletions engines/ep/src/collections/collections_types.h
View file
Expand Up @@ -86,24 +86,53 @@ static_assert(sizeof(ManifestUidNetworkOrder) == 8,
*/
ManifestUid makeManifestUid(std::string_view uid);

/// Collection metering yes/no
enum class Metered : bool { Yes, No };

std::string to_string(Metered);
std::ostream& operator<<(std::ostream&, Metered);

/**
* The metadata of a single collection. This represents the data we persist
* in KVStore meta and is used to communicate back to kv from KVStore
*
* Default construction yields the default collection
*/
struct CollectionMetaData {
CollectionMetaData() = default;
CollectionMetaData(ScopeID sid,
CollectionID cid,
std::string_view name,
cb::ExpiryLimit maxTtl,
Metered metered,
CanDeduplicate canDeduplicate)
: sid(sid),
cid(cid),
name(name),
maxTtl(maxTtl),
metered(metered),
canDeduplicate(canDeduplicate) {
}

ScopeID sid{ScopeID::Default}; // The scope that the collection belongs to
CollectionID cid{CollectionID::Default}; // The collection's ID
std::string name{DefaultCollectionName}; // The collection's name
cb::ExpiryLimit maxTtl{}; // The collection's maxTTL
Metered metered{Metered::Yes};
CanDeduplicate canDeduplicate{CanDeduplicate::Yes};

bool operator==(const CollectionMetaData& other) const {
return sid == other.sid && cid == other.cid && name == other.name &&
maxTtl == other.maxTtl;
maxTtl == other.maxTtl && metered == other.metered &&
canDeduplicate == other.canDeduplicate;
}

bool operator!=(const CollectionMetaData& other) const {
return !(*this == other);
}
};

std::string to_string(const CollectionMetaData&);
std::ostream& operator<<(std::ostream& os, const CollectionMetaData& meta);

/**
Expand Down Expand Up @@ -171,12 +200,6 @@ using DataLimit = std::optional<size_t>;

static const DataLimit NoDataLimit{};

/// Collection metering yes/no
enum class Metered : bool { Yes, No };

std::string to_string(Metered);
std::ostream& operator<<(std::ostream&, Metered);

namespace VB {
enum class ManifestUpdateStatus {
Success,
Expand Down
7 changes: 6 additions & 1 deletion engines/ep/src/collections/kvstore.cc
View file
Expand Up @@ -84,7 +84,12 @@ Collections::KVStore::Manifest decodeManifest(cb::const_byte_buffer manifest,
Collections::CollectionMetaData{entry->scopeId(),
entry->collectionId(),
entry->name()->str(),
maxTtl}});
maxTtl,
// metered: not persisted
// correct value comes from
// bucket manifest
Collections::Metered::Yes,
CanDeduplicate::Yes}});
}
} else {
// Nothing on disk - the default collection is assumed
Expand Down
4 changes: 2 additions & 2 deletions engines/ep/src/collections/kvstore.h
View file
Expand Up @@ -78,8 +78,8 @@ struct Manifest {
* - no dropped collections
*/
explicit Manifest(Default)
: collections{{CollectionID::Default, {0, {}}}},
scopes{{ScopeID::Default, {0, {}}}} {
: collections{{0 /*seqno*/, CollectionMetaData{}}},
scopes{{0 /*seqno*/, ScopeMetaData{}}} {
}

/**
Expand Down
22 changes: 11 additions & 11 deletions engines/ep/src/collections/manager.cc
View file
Expand Up @@ -282,21 +282,21 @@ std::pair<uint64_t, std::optional<ScopeID>> Collections::Manager::getScopeID(
current->getUid(), current->getScopeID(cid));
}

std::pair<uint64_t, std::optional<Collections::CollectionEntry>>
std::pair<uint64_t, std::optional<Collections::CollectionMetaData>>
Collections::Manager::getCollectionEntry(CollectionID cid) const {
// 'shortcut' For the default collection, just return the default scope.
// If the default collection was deleted the vbucket will have the final say
// but for this interface allow this without taking the rlock.
if (cid.isDefaultCollection()) {
// Allow the default collection in the default scope...
return std::make_pair<uint64_t,
std::optional<Collections::CollectionEntry>>(
0, Collections::DefaultCollectionEntry);
std::optional<Collections::CollectionMetaData>>(
0, Collections::CollectionMetaData{});
}

auto current = currentManifest.rlock();
return std::make_pair<uint64_t,
std::optional<Collections::CollectionEntry>>(
std::optional<Collections::CollectionMetaData>>(
current->getUid(), current->getCollectionEntry(cid));
}

Expand Down Expand Up @@ -423,7 +423,7 @@ class AllCollectionsGetStatsVBucketVisitor : public VBucketVisitor {
class CollectionsGetStatsVBucketVisitor : public VBucketVisitor {
public:
explicit CollectionsGetStatsVBucketVisitor(
const std::vector<Collections::CollectionEntry>& collections)
const std::vector<Collections::CollectionMetaData>& collections)
: collections(collections) {
for (const auto& entry : collections) {
summary.emplace(entry.cid, Collections::AccumulatedStats{});
Expand All @@ -436,7 +436,7 @@ class CollectionsGetStatsVBucketVisitor : public VBucketVisitor {
}
}

const std::vector<Collections::CollectionEntry>& collections;
const std::vector<Collections::CollectionMetaData>& collections;
Collections::Summary summary;
};

Expand Down Expand Up @@ -630,7 +630,7 @@ cb::EngineErrorGetCollectionIDResult Collections::Manager::doOneCollectionStats(
// Take a copy of the two items needed from the manifest and then release
// the lock before vbucket visiting.
std::string scopeName;
Collections::CollectionEntry entry;
Collections::CollectionMetaData entry;
{
auto current = bucket.getCollectionsManager().currentManifest.rlock();
auto collectionItr = current->findCollection(res.getCollectionId());
Expand Down Expand Up @@ -785,7 +785,7 @@ cb::EngineErrorGetScopeIDResult Collections::Manager::doOneScopeStats(
// Take a copy of the two items needed from the manifest and then release
// the lock before vbucket visiting.
std::string scopeName;
std::vector<Collections::CollectionEntry> scopeCollections;
std::vector<Collections::CollectionMetaData> scopeCollections;
{
auto current = bucket.getCollectionsManager().currentManifest.rlock();
auto scopeItr = current->findScope(res.getScopeId());
Expand Down Expand Up @@ -842,7 +842,7 @@ Collections::CachedStats Collections::Manager::getPerCollectionStats(
}

Collections::CachedStats Collections::Manager::getPerCollectionStats(
const std::vector<Collections::CollectionEntry>& collections,
const std::vector<Collections::CollectionMetaData>& collections,
KVBucket& bucket) {
// Gather per-vbucket stats for the collections of interest
CollectionsGetStatsVBucketVisitor visitor{collections};
Expand All @@ -868,7 +868,7 @@ Collections::CachedStats::CachedStats(

void Collections::CachedStats::addStatsForCollection(
std::string_view scopeName,
const CollectionEntry& collection,
const CollectionMetaData& collection,
const BucketStatCollector& collector) {
auto collectionC = collector.forScope(scopeName, collection.sid)
.forCollection(collection.name, collection.cid);
Expand All @@ -889,7 +889,7 @@ void Collections::CachedStats::addStatsForCollection(
void Collections::CachedStats::addStatsForScope(
ScopeID sid,
std::string_view scopeName,
const std::vector<Collections::CollectionEntry>& scopeCollections,
const std::vector<Collections::CollectionMetaData>& scopeCollections,
const BucketStatCollector& collector) {
auto scopeC = collector.forScope(scopeName, sid);
std::vector<CollectionID> collections;
Expand Down
17 changes: 9 additions & 8 deletions engines/ep/src/collections/manager.h
View file
Expand Up @@ -57,7 +57,7 @@ class CachedStats {
* @param collector stat collector to which stats will be added
*/
void addStatsForCollection(std::string_view scopeName,
const CollectionEntry& collection,
const CollectionMetaData& collection,
const BucketStatCollector& collector);

/**
Expand All @@ -68,11 +68,11 @@ class CachedStats {
* @param scopeCollections All collections in the scope
* @param collector stat collector to which stats will be added
*/
void addStatsForScope(
ScopeID sid,
std::string_view scopeName,
const std::vector<Collections::CollectionEntry>& scopeCollections,
const BucketStatCollector& collector);
void addStatsForScope(ScopeID sid,
std::string_view scopeName,
const std::vector<Collections::CollectionMetaData>&
scopeCollections,
const BucketStatCollector& collector);

private:
/**
Expand Down Expand Up @@ -169,7 +169,7 @@ class Manager {
* @return pair of manifest-id (which was used in the search) and an
* optional. If the lookup was successful the optional contains the entry
*/
std::pair<uint64_t, std::optional<CollectionEntry>> getCollectionEntry(
std::pair<uint64_t, std::optional<CollectionMetaData>> getCollectionEntry(
CollectionID) const;

/**
Expand Down Expand Up @@ -370,7 +370,8 @@ class Manager {
* @return copy of the stats to use to format stats for a request
*/
static CachedStats getPerCollectionStats(
const std::vector<CollectionEntry>& collections, KVBucket& bucket);
const std::vector<CollectionMetaData>& collections,
KVBucket& bucket);

/**
* validate the path is correctly formed for get_collection_id.
Expand Down
48 changes: 13 additions & 35 deletions engines/ep/src/collections/manifest.cc
View file
Expand Up @@ -175,7 +175,7 @@ Manifest::Manifest(std::string_view json, size_t numVbuckets)
}
}

std::vector<CollectionEntry> scopeCollections = {};
std::vector<CollectionMetaData> scopeCollections;

// Read the collections within this scope
auto collections =
Expand Down Expand Up @@ -262,12 +262,12 @@ Manifest::Manifest(std::string_view json, size_t numVbuckets)
}

enableDefaultCollection(cidValue);
scopeCollections.push_back(CollectionEntry{cidValue,
cnameValue,
maxTtl,
sidValue,
collectionCanDeduplicate,
meteredState});
scopeCollections.emplace_back(sidValue,
cidValue,
cnameValue,
maxTtl,
meteredState,
collectionCanDeduplicate);
}

// Check for limits - only support for data_size
Expand Down Expand Up @@ -525,7 +525,7 @@ Manifest::Manifest(std::string_view flatbufferData, Manifest::FlatBuffers tag)
uid = manifest->uid();

for (const Collections::Persist::Scope* scope : *manifest->scopes()) {
std::vector<CollectionEntry> scopeCollections;
std::vector<CollectionMetaData> scopeCollections;

for (const Collections::Persist::Collection* collection :
*scope->collections()) {
Expand All @@ -536,13 +536,13 @@ Manifest::Manifest(std::string_view flatbufferData, Manifest::FlatBuffers tag)
}

enableDefaultCollection(cid);
scopeCollections.push_back(CollectionEntry{
scopeCollections.emplace_back(
ScopeID{scope->scopeId()},
cid,
collection->name()->str(),
maxTtl,
scope->scopeId(),
getCanDeduplicateFromHistory(collection->history()),
collection->metered() ? Metered::Yes : Metered::No});
collection->metered() ? Metered::Yes : Metered::No,
getCanDeduplicateFromHistory(collection->history()));
}

std::optional<size_t> dataSize;
Expand Down Expand Up @@ -744,7 +744,7 @@ std::optional<Metered> Manifest::isMetered(CollectionID cid) const {
return {};
}

std::optional<CollectionEntry> Manifest::getCollectionEntry(
std::optional<CollectionMetaData> Manifest::getCollectionEntry(
CollectionID cid) const {
auto itr = collections.find(cid);
if (itr != collections.end()) {
Expand All @@ -765,12 +765,6 @@ void Manifest::dump() const {
std::cerr << *this << std::endl;
}

bool CollectionEntry::operator==(const CollectionEntry& other) const {
return cid == other.cid && name == other.name && sid == other.sid &&
maxTtl == other.maxTtl && metered == other.metered &&
canDeduplicate == other.canDeduplicate;
}

bool Scope::operator==(const Scope& other) const {
bool equal = name == other.name &&
collections.size() == other.collections.size() &&
Expand Down Expand Up @@ -931,22 +925,6 @@ std::ostream& operator<<(std::ostream& os, const Manifest& manifest) {
return os;
}

std::string to_string(const CollectionEntry& collection) {
return fmt::format(
"cid:{}, name:{}, ttl:{{{}, {}}}, sid:{}, {}, {}",
collection.cid.to_string(),
collection.name,
collection.maxTtl.has_value(),
collection.maxTtl.value_or(std::chrono::seconds(0)).count(),
collection.sid.to_string(),
collection.canDeduplicate,
collection.metered);
}

std::ostream& operator<<(std::ostream& os, const CollectionEntry& collection) {
return os << to_string(collection);
}

std::string to_string(const Scope& scope) {
// not descending into the collections vector as caller can choose how to
// space that out.
Expand Down

0 comments on commit 759684a

Please sign in to comment.