Skip to content

Commit

Permalink
Merge 67cc587 into a9d4304
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChristopherHogan
ChristopherHogan committed Dec 14, 2021
2 parents a9d4304 + 67cc587 commit 3ae58f1
Show file tree
Hide file tree
Showing 7 changed files with 30 additions and 107 deletions.
10 changes: 4 additions & 6 deletions adapter/src/hermes/adapter/mpiio/mpiio.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -583,10 +583,9 @@ int HERMES_DECL(MPI_File_close)(MPI_File *fh) {
}
}
}
auto file_mapping = hapi::FileMappingTrait(filename, offset_map,
nullptr, NULL, NULL);
bool flush_synchronously = true;
hapi::PersistTrait persist_trait(file_mapping, flush_synchronously);
hapi::PersistTrait persist_trait(filename, offset_map,
flush_synchronously);
file_vbucket.Attach(&persist_trait, ctx);
file_vbucket.Destroy(ctx);

Expand Down Expand Up @@ -1103,10 +1102,9 @@ int HERMES_DECL(MPI_File_sync)(MPI_File fh) {
}
}
}
auto file_mapping = hapi::FileMappingTrait(filename, offset_map,
nullptr, NULL, NULL);
bool flush_synchronously = true;
hapi::PersistTrait persist_trait(file_mapping, flush_synchronously);
hapi::PersistTrait persist_trait(filename, offset_map,
flush_synchronously);
file_vbucket.Attach(&persist_trait);
file_vbucket.Destroy();
for (const auto &vbucket : existing.first.st_vbuckets) {
Expand Down
11 changes: 4 additions & 7 deletions adapter/src/hermes/adapter/posix/posix.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -792,10 +792,9 @@ int HERMES_DECL(fsync)(int fd) {
offset_map.emplace(blob_name, page_index * kPageSize);
}
}
auto file_mapping = hapi::FileMappingTrait(filename, offset_map,
nullptr, NULL, NULL);
bool flush_synchronously = true;
hapi::PersistTrait persist_trait(file_mapping, flush_synchronously);
hapi::PersistTrait persist_trait(filename, offset_map,
flush_synchronously);
file_vbucket.Attach(&persist_trait, ctx);
file_vbucket.Destroy(ctx);
existing.first.st_blobs.clear();
Expand Down Expand Up @@ -849,11 +848,9 @@ int HERMES_DECL(close)(int fd) {
offset_map.emplace(blob_name, page_index * kPageSize);
}
}
auto file_mapping =
hapi::FileMappingTrait(filename, offset_map, nullptr, NULL,
NULL);
bool flush_synchronously = true;
hapi::PersistTrait persist_trait(file_mapping, flush_synchronously);
hapi::PersistTrait persist_trait(filename, offset_map,
flush_synchronously);
file_vbucket.Attach(&persist_trait, ctx);
file_vbucket.Destroy(ctx);
existing.first.st_blobs.clear();
Expand Down
17 changes: 6 additions & 11 deletions adapter/src/hermes/adapter/stdio/stdio.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,8 @@ FILE *simple_open(FILE *ret, const std::string &path_str, const char *mode) {
stat.st_vbkt =
std::make_shared<hapi::VBucket>(path_str, mdm->GetHermes());
auto offset_map = std::unordered_map<std::string, u64>();
hapi::FileMappingTrait mapping_trait(path_str, offset_map,
NULL, NULL, NULL);
stat.st_persist =
std::make_shared<hapi::PersistTrait>(mapping_trait, false);
std::make_shared<hapi::PersistTrait>(path_str, offset_map, false);
stat.st_vbkt->Attach(stat.st_persist.get());
}

Expand Down Expand Up @@ -343,8 +341,7 @@ size_t write_internal(AdapterStat &stat, const void *ptr, size_t total_size,
hapi::Trait *trait = stat.st_vbkt->GetTrait(hapi::TraitType::PERSIST);
if (trait) {
hapi::PersistTrait *persist_trait = (hapi::PersistTrait *)trait;
persist_trait->file_mapping.offset_map.emplace(hinfo.blob_name_,
offset);
persist_trait->offset_map.emplace(hinfo.blob_name_, offset);
}

stat.st_vbkt->Link(hinfo.blob_name_, filename);
Expand Down Expand Up @@ -587,10 +584,9 @@ int HERMES_DECL(fflush)(FILE *fp) {
auto page_index = std::stol(blob_name) - 1;
offset_map.emplace(blob_name, page_index * kPageSize);
}
auto file_mapping = hapi::FileMappingTrait(filename, offset_map,
nullptr, NULL, NULL);
bool flush_synchronously = true;
hapi::PersistTrait persist_trait(file_mapping, flush_synchronously);
hapi::PersistTrait persist_trait(filename, offset_map,
flush_synchronously);
file_vbucket.Attach(&persist_trait);
existing.first.st_blobs.clear();
file_vbucket.Destroy();
Expand Down Expand Up @@ -637,10 +633,9 @@ int HERMES_DECL(fclose)(FILE *fp) {
offset_map.emplace(blob_name, page_index * kPageSize);
}
}
auto file_mapping =
hapi::FileMappingTrait(filename, offset_map, nullptr, NULL, NULL);
bool flush_synchronously = true;
hapi::PersistTrait persist_trait(file_mapping, flush_synchronously);
hapi::PersistTrait persist_trait(filename, offset_map,
flush_synchronously);
file_vbucket.Attach(&persist_trait);
existing.first.st_blobs.clear();
INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename);
Expand Down
63 changes: 8 additions & 55 deletions src/api/traits.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,60 +27,13 @@ Trait::Trait(TraitID id, TraitIdArray conflict_traits, TraitType type)
onLinkFn(nullptr),
onUnlinkFn(nullptr) {}

FileMappingTrait::FileMappingTrait(
const std::string &filename,
std::unordered_map<std::string, u64> &offset_map,
FILE *fh,
OnLinkCallback flush_cb, OnLinkCallback load_cb)
: Trait(HERMES_FILE_TRAIT, TraitIdArray(), TraitType::FILE_MAPPING),
flush_cb(flush_cb),
load_cb(load_cb),
filename(filename),
offset_map(offset_map),
fh(fh) {
this->onAttachFn = std::bind(&FileMappingTrait::onAttach, this,
std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
this->onDetachFn = std::bind(&FileMappingTrait::onDetach, this,
std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
this->onLinkFn = std::bind(&FileMappingTrait::onLink, this,
std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
this->onUnlinkFn = std::bind(&FileMappingTrait::onUnlink, this,
std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
}

void FileMappingTrait::onAttach(HermesPtr hermes, VBucketID id, Trait *trait) {
(void)hermes;
(void)id;
(void)trait;
}

void FileMappingTrait::onDetach(HermesPtr hermes, VBucketID id, Trait *trait) {
(void)hermes;
(void)id;
(void)trait;
}

void FileMappingTrait::onLink(HermesPtr hermes, TraitInput &input,
Trait *trait) {
(void)hermes;
(void)input;
(void)trait;
}

void FileMappingTrait::onUnlink(HermesPtr hermes, TraitInput &input,
Trait *trait) {
(void)hermes;
(void)input;
(void)trait;
}
using OffsetMap = std::unordered_map<std::string, u64>;

PersistTrait::PersistTrait(FileMappingTrait mapping, bool synchronous)
PersistTrait::PersistTrait(const std::string &filename,
const OffsetMap &offset_map,
bool synchronous)
: Trait(HERMES_PERSIST_TRAIT, TraitIdArray(), TraitType::PERSIST),
file_mapping(mapping), synchronous(synchronous) {
filename(filename), offset_map(offset_map), synchronous(synchronous) {
this->onAttachFn = std::bind(&PersistTrait::onAttach, this,
std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
Expand Down Expand Up @@ -118,14 +71,14 @@ void PersistTrait::onDetach(HermesPtr hermes, VBucketID id, Trait *trait) {

void PersistTrait::onLink(HermesPtr hermes, TraitInput &input, Trait *trait) {
PersistTrait *persist_trait = (PersistTrait *)trait;
auto iter = persist_trait->file_mapping.offset_map.find(input.blob_name);
auto iter = persist_trait->offset_map.find(input.blob_name);

if (iter != persist_trait->file_mapping.offset_map.end()) {
if (iter != persist_trait->offset_map.end()) {
SharedMemoryContext *context = &hermes->context_;
RpcContext *rpc = &hermes->rpc_;
BucketID bucket_id = GetBucketId(context, rpc, input.bucket_name.c_str());
BlobID blob_id = GetBlobId(context, rpc, input.blob_name, bucket_id, true);
std::string filename = persist_trait->file_mapping.filename;
std::string filename = persist_trait->filename;
u64 offset = iter->second;

if (synchronous) {
Expand Down
25 changes: 4 additions & 21 deletions src/api/traits.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,34 +59,17 @@ struct Trait {
Trait(TraitID id, TraitIdArray conflict_traits, TraitType type);
};

#define HERMES_FILE_TRAIT 10
#define HERMES_PERSIST_TRAIT 11

/** File mapping trait */
struct FileMappingTrait : public Trait {
OnLinkCallback flush_cb;
OnLinkCallback load_cb;
std::string filename;
std::unordered_map<std::string, u64> offset_map;
FILE *fh;

FileMappingTrait() {}
FileMappingTrait(const std::string &filename,
std::unordered_map<std::string, u64> &offset_map, FILE *fh,
OnLinkCallback flush_cb, OnLinkCallback load_cb);
void onAttach(HermesPtr hermes, VBucketID id, Trait *trait);
void onDetach(HermesPtr hermes, VBucketID id, Trait *trait);
void onLink(HermesPtr hermes, TraitInput &blob, Trait *trait);
void onUnlink(HermesPtr hermes, TraitInput &blob, Trait *trait);
};

/** (File) Persistence trait */
struct PersistTrait : public Trait {
FileMappingTrait file_mapping;
std::string filename;
std::unordered_map<std::string, u64> offset_map;
bool synchronous;

explicit PersistTrait(bool synchronous);
explicit PersistTrait(FileMappingTrait mapping,
explicit PersistTrait(const std::string &filename,
const std::unordered_map<std::string, u64> &offset_map,
bool synchronous = false);

void onAttach(HermesPtr hermes, VBucketID id, Trait *trait);
Expand Down
6 changes: 2 additions & 4 deletions test/buffer_organizer_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,7 @@ void TestBackgroundFlush() {
hapi::Bucket bkt(bkt_name, hermes);
hapi::VBucket vbkt(bkt_name, hermes);

hapi::FileMappingTrait mapping;
mapping.filename = final_destination;
hapi::PersistTrait persist_trait(mapping, false);
hapi::PersistTrait persist_trait(final_destination, {}, false);
vbkt.Attach(&persist_trait);

std::vector<u8> data(iters);
Expand All @@ -86,7 +84,7 @@ void TestBackgroundFlush() {
std::string blob_name = std::to_string(i);
bkt.Put(blob_name, blob);

persist_trait.file_mapping.offset_map.emplace(blob_name, i * io_size);
persist_trait.offset_map.emplace(blob_name, i * io_size);
vbkt.Link(blob_name, bkt_name);
}

Expand Down
5 changes: 2 additions & 3 deletions test/trait_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,9 @@ TEST_CASE("CustomTrait",
file_vbucket.Link(blob_name, args.filename);
offset_map.emplace(blob_name, std::stol(blob_name) * info.FILE_PAGE);
}
auto fm_trait = hapi::FileMappingTrait(fullpath_str, offset_map,
nullptr, NULL, NULL);
bool flush_synchronously = true;
hapi::PersistTrait persist_trait(fm_trait, flush_synchronously);
hapi::PersistTrait persist_trait(fullpath_str, offset_map,
flush_synchronously);
file_vbucket.Attach(&persist_trait);
file_vbucket.Destroy();
file_bucket.Destroy();
Expand Down

0 comments on commit 3ae58f1

Please sign in to comment.