Skip to content

Commit 22dc762

Browse files
SquadrickSquadrick
authored
Implement custom copier for binary IPC (#16)
1 parent 09fe8fe commit 22dc762

File tree

7 files changed

+140
-43
lines changed

7 files changed

+140
-43
lines changed

‎include/shadesmar/memory/copier.h‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/* MIT License
2+
3+
Copyright (c) 2020 Dheeraj R Reddy
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.
22+
==============================================================================*/
23+
24+
#ifndef INCLUDE_SHADESMAR_MEMORY_COPIER_H_
25+
#define INCLUDE_SHADESMAR_MEMORY_COPIER_H_
26+
27+
#include <cstring>
28+
29+
namespace shm::memory {
30+
class Copier {
31+
public:
32+
virtual void *alloc(size_t) = 0;
33+
virtual void dealloc(void *) = 0;
34+
virtual void shm_to_user(void *, void *, size_t) = 0;
35+
virtual void user_to_shm(void *, void *, size_t) = 0;
36+
};
37+
38+
class DefaultCopier : public Copier {
39+
public:
40+
void *alloc(size_t size) override { return malloc(size); }
41+
42+
void dealloc(void *ptr) override { free(ptr); }
43+
44+
void shm_to_user(void *dst, void *src, size_t size) override {
45+
std::memcpy(dst, src, size);
46+
}
47+
48+
void user_to_shm(void *dst, void *src, size_t size) override {
49+
std::memcpy(dst, src, size);
50+
}
51+
};
52+
53+
} // namespace shm::memory
54+
55+
#endif // INCLUDE_SHADESMAR_MEMORY_COPIER_H_

‎include/shadesmar/memory/dragons.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@ SOFTWARE.
3232
#include <thread>
3333
#include <vector>
3434

35+
#include "shadesmar/memory/copier.h"
36+
3537
namespace shm::memory::dragons {
38+
3639
static inline void *_rep_movsb(void *d, const void *s, size_t n) {
3740
// Slower than using regular `memcpy`
3841
asm volatile("rep movsb"

‎include/shadesmar/memory/memory.h‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,21 +83,25 @@ uint8_t *create_memory_segment(const std::string &name, size_t size,
8383
return static_cast<uint8_t *>(ptr);
8484
}
8585

86+
struct Ptr {
87+
void *ptr;
88+
size_t size;
89+
bool free;
90+
91+
Ptr() : ptr(nullptr), size(0), free(false) {}
92+
93+
void no_delete() { free = false; }
94+
};
95+
8696
struct Element {
8797
size_t size;
8898
std::atomic<bool> empty{};
8999
managed_shared_memory::handle_t addr_hdl;
90100

91-
Element() {
92-
size = 0;
93-
empty.store(true);
94-
addr_hdl = 0;
95-
}
101+
Element() : size(0), addr_hdl(0) { empty.store(true); }
96102

97-
Element(const Element &elem) {
98-
size = elem.size;
103+
Element(const Element &elem) : size(elem.size), addr_hdl(elem.addr_hdl) {
99104
empty.store(elem.empty.load());
100-
addr_hdl = elem.addr_hdl;
101105
}
102106
};
103107

‎include/shadesmar/pubsub/publisher.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ SOFTWARE.
3333

3434
#include <msgpack.hpp>
3535

36+
#include "shadesmar/memory/copier.h"
3637
#include "shadesmar/message.h"
3738
#include "shadesmar/pubsub/topic.h"
3839

@@ -41,7 +42,7 @@ namespace shm::pubsub {
4142
template <uint32_t queue_size>
4243
class PublisherBin {
4344
public:
44-
explicit PublisherBin(std::string topic_name);
45+
explicit PublisherBin(std::string topic_name, memory::Copier *copier);
4546
bool publish(void *data, size_t size);
4647

4748
private:
@@ -50,8 +51,9 @@ class PublisherBin {
5051
};
5152

5253
template <uint32_t queue_size>
53-
PublisherBin<queue_size>::PublisherBin(std::string topic_name)
54-
: topic_name_(topic_name), topic_(Topic<queue_size>(topic_name)) {}
54+
PublisherBin<queue_size>::PublisherBin(std::string topic_name,
55+
memory::Copier *copier)
56+
: topic_name_(topic_name), topic_(Topic<queue_size>(topic_name, copier)) {}
5557

5658
template <uint32_t queue_size>
5759
bool PublisherBin<queue_size>::publish(void *data, size_t size) {

‎include/shadesmar/pubsub/subscriber.h‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ SOFTWARE.
3434
#include <thread>
3535
#include <utility>
3636

37+
#include "shadesmar/memory/copier.h"
3738
#include "shadesmar/message.h"
3839
#include "shadesmar/pubsub/topic.h"
3940

@@ -48,7 +49,9 @@ class SubscriberBase {
4849
virtual void _subscribe() = 0;
4950

5051
protected:
51-
SubscriberBase(std::string topic_name, bool extra_copy);
52+
SubscriberBase(std::string topic_name, memory::Copier *copier,
53+
bool extra_copy);
54+
5255
std::string topic_name_;
5356
std::unique_ptr<Topic<queue_size>> topic;
5457
std::atomic<uint32_t> counter{0};
@@ -57,14 +60,13 @@ class SubscriberBase {
5760
template <uint32_t queue_size>
5861
class SubscriberBin : public SubscriberBase<queue_size> {
5962
public:
60-
SubscriberBin(
61-
const std::string &topic_name,
62-
std::function<void(std::unique_ptr<uint8_t[]> &, size_t)> callback)
63-
: SubscriberBase<queue_size>(topic_name, false),
63+
SubscriberBin(const std::string &topic_name, memory::Copier *copier,
64+
std::function<void(memory::Ptr *)> callback)
65+
: SubscriberBase<queue_size>(topic_name, copier, false),
6466
callback_(std::move(callback)) {}
6567

6668
private:
67-
std::function<void(std::unique_ptr<uint8_t[]> &, size_t)> callback_;
69+
std::function<void(memory::Ptr *)> callback_;
6870
void _subscribe();
6971
};
7072

@@ -77,7 +79,7 @@ class Subscriber : public SubscriberBase<queue_size> {
7779
Subscriber(const std::string &topic_name,
7880
std::function<void(const std::shared_ptr<msgT> &)> callback,
7981
bool extra_copy = false)
80-
: SubscriberBase<queue_size>(topic_name, extra_copy),
82+
: SubscriberBase<queue_size>(topic_name, nullptr, extra_copy),
8183
callback_(std::move(callback)) {}
8284

8385
private:
@@ -87,10 +89,11 @@ class Subscriber : public SubscriberBase<queue_size> {
8789

8890
template <uint32_t queue_size>
8991
SubscriberBase<queue_size>::SubscriberBase(std::string topic_name,
92+
memory::Copier *copier,
9093
bool extra_copy)
9194
: topic_name_(std::move(topic_name)) {
9295
#if __cplusplus >= 201703L
93-
topic = std::make_unique<Topic<queue_size>>(topic_name_, extra_copy);
96+
topic = std::make_unique<Topic<queue_size>>(topic_name_, copier, extra_copy);
9497
#else
9598
topic = std::unique_ptr<Topic<queue_size>>(new Topic<queue_size>(
9699
std::forward<std::string>(topic_name_, extra_copy)));
@@ -138,12 +141,16 @@ void SubscriberBase<queue_size>::spin() {
138141

139142
template <uint32_t queue_size>
140143
void SubscriberBin<queue_size>::_subscribe() {
141-
std::unique_ptr<uint8_t[]> msg;
142-
size_t size = 0;
144+
memory::Ptr ptr;
145+
ptr.free = true;
146+
147+
if (!this->topic->read(&ptr, this->counter)) return;
143148

144-
if (!this->topic->read(msg, &size, this->counter)) return;
149+
callback_(&ptr);
145150

146-
callback_(msg, size);
151+
if (ptr.free) {
152+
this->topic->copier()->dealloc(ptr.ptr);
153+
}
147154
}
148155

149156
template <typename msgT, uint32_t queue_size>

‎include/shadesmar/pubsub/topic.h‎

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ SOFTWARE.
3535

3636
#include "shadesmar/concurrency/scope.h"
3737
#include "shadesmar/macros.h"
38+
#include "shadesmar/memory/copier.h"
3839
#include "shadesmar/memory/memory.h"
3940

4041
namespace shm::pubsub {
@@ -61,8 +62,13 @@ class Topic : public memory::Memory<_TopicElem<LockT>, queue_size> {
6162
"queue_size must be power of two");
6263

6364
public:
65+
Topic(const std::string &topic, memory::Copier *copier, bool copy = false)
66+
: memory::Memory<TopicElem, queue_size>(topic),
67+
copy_(copy),
68+
copier_(copier) {}
69+
6470
explicit Topic(const std::string &topic, bool copy = false)
65-
: memory::Memory<TopicElem, queue_size>(topic), copy_(copy) {}
71+
: Topic(topic, nullptr, copy) {}
6672

6773
bool write(void *data, size_t size) {
6874
/*
@@ -102,14 +108,13 @@ class Topic : public memory::Memory<_TopicElem<LockT>, queue_size> {
102108
return _read_without_copy(oh, elem);
103109
}
104110

105-
bool read(std::unique_ptr<uint8_t[]> &msg, size_t *size, // NOLINT
106-
uint32_t pos) {
111+
bool read(memory::Ptr *ptr, uint32_t pos) {
107112
/*
108113
* Read into a raw array. We pass `size` as a reference to store
109114
* the size of the message.
110115
*/
111116
TopicElem *elem = &(this->shared_queue_->elements[pos & (queue_size - 1)]);
112-
return _read_bin(msg, size, elem);
117+
return _read_bin(ptr, elem);
113118
}
114119

115120
inline __attribute__((always_inline)) uint32_t fetch_add_counter() {
@@ -124,6 +129,10 @@ class Topic : public memory::Memory<_TopicElem<LockT>, queue_size> {
124129
this->shared_queue_->counter.fetch_add(1);
125130
}
126131

132+
inline __attribute__((always_inline)) memory::Copier *copier() const {
133+
return copier_;
134+
}
135+
127136
private:
128137
bool _write_rcu(void *data, size_t size, TopicElem *elem) {
129138
/*
@@ -139,7 +148,12 @@ class Topic : public memory::Memory<_TopicElem<LockT>, queue_size> {
139148
if (new_addr == nullptr) {
140149
return false;
141150
}
142-
std::memcpy(new_addr, data, size);
151+
152+
if (copier_ == nullptr) {
153+
std::memcpy(new_addr, data, size);
154+
} else {
155+
copier_->user_to_shm(new_addr, data, size);
156+
}
143157

144158
void *old_addr;
145159
bool prev_empty;
@@ -195,18 +209,19 @@ class Topic : public memory::Memory<_TopicElem<LockT>, queue_size> {
195209
* So we bite the cost of the extra function call before checking
196210
* for emptiness.
197211
*/
198-
auto src = std::unique_ptr<uint8_t[]>(new uint8_t[elem->size]);
199-
size_t size;
200212

201-
if (_read_bin(src, &size, elem)) {
202-
*oh = msgpack::unpack(reinterpret_cast<const char *>(src.get()), size);
213+
memory::Ptr ptr;
214+
215+
if (_read_bin(&ptr, elem)) {
216+
*oh = msgpack::unpack(reinterpret_cast<const char *>(ptr.ptr), ptr.size);
217+
free(ptr.ptr);
203218
return true;
204219
}
220+
205221
return false;
206222
}
207223

208-
bool _read_bin(std::unique_ptr<uint8_t[]> &msg, size_t *size, // NOLINT
209-
TopicElem *elem) {
224+
bool _read_bin(memory::Ptr *ptr, TopicElem *elem) {
210225
/*
211226
* Code path:
212227
* 1. Acquire sharable lock
@@ -218,14 +233,21 @@ class Topic : public memory::Memory<_TopicElem<LockT>, queue_size> {
218233

219234
if (elem->empty) return false;
220235

221-
*size = elem->size;
222-
msg = std::unique_ptr<uint8_t[]>(new uint8_t[*size]);
223236
auto *dst = this->raw_buf_->get_address_from_handle(elem->addr_hdl);
224-
std::memcpy(msg.get(), dst, *size);
237+
238+
ptr->size = elem->size;
239+
if (copier_ != nullptr) {
240+
ptr->ptr = copier_->alloc(ptr->size);
241+
copier_->shm_to_user(ptr->ptr, dst, ptr->size);
242+
} else {
243+
ptr->ptr = malloc(ptr->size);
244+
std::memcpy(ptr->ptr, dst, ptr->size);
245+
}
225246

226247
return true;
227248
}
228249

250+
memory::Copier *copier_;
229251
bool copy_{};
230252
};
231253
} // namespace shm::pubsub

‎test/pubsub_bin_test.cpp‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2121
SOFTWARE.
2222
==============================================================================*/
2323

24-
#include <shadesmar/pubsub/publisher.h>
25-
#include <shadesmar/pubsub/subscriber.h>
2624
#include <chrono>
2725
#include <iostream>
2826
#include <numeric>
2927

28+
#include "shadesmar/memory/copier.h"
29+
#include "shadesmar/pubsub/publisher.h"
30+
#include "shadesmar/pubsub/subscriber.h"
31+
3032
const char topic[] = "raw_benchmark_topic";
3133
const int QUEUE_SIZE = 16;
3234
const int SECONDS = 10;
@@ -60,8 +62,8 @@ double get_stddev(const std::vector<T> &v) {
6062
return stddev;
6163
}
6264

63-
void callback(const std::unique_ptr<uint8_t[]> &data, uint32_t size) {
64-
auto *msg = reinterpret_cast<Message *>(data.get());
65+
void callback(shm::memory::Ptr *shm_ptr) {
66+
auto *msg = reinterpret_cast<Message *>(shm_ptr->ptr);
6567
++count;
6668
++total_count;
6769
lag += std::chrono::duration_cast<TIMESCALE>(
@@ -75,7 +77,8 @@ int main() {
7577
std::vector<int> counts;
7678
std::vector<double> lags;
7779
std::this_thread::sleep_for(std::chrono::seconds(1));
78-
shm::pubsub::SubscriberBin<QUEUE_SIZE> sub(topic, callback);
80+
shm::memory::DefaultCopier cpy;
81+
shm::pubsub::SubscriberBin<QUEUE_SIZE> sub(topic, &cpy, callback);
7982
auto start = std::chrono::system_clock::now();
8083
int seconds = 0;
8184
while (true) {
@@ -114,7 +117,8 @@ int main() {
114117
std::cout << "Lag: " << mean_lag << " ± " << stdd_lag << TIMESCALE_NAME
115118
<< std::endl;
116119
} else {
117-
shm::pubsub::PublisherBin<QUEUE_SIZE> pub(topic);
120+
shm::memory::DefaultCopier cpy;
121+
shm::pubsub::PublisherBin<QUEUE_SIZE> pub(topic, &cpy);
118122

119123
Message *msg = reinterpret_cast<Message *>(malloc(VECTOR_SIZE));
120124

0 commit comments

Comments
 (0)