Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 448 lines (386 sloc) 11.688 kB
236b14f @avsm add licenses
authored
1 /*
2 Measure latency of IPC using tcp sockets
3
4 Copyright (c) 2011 Steven Smith <sos22@cam.ac.uk>
5
6 Permission is hereby granted, free of charge, to any person
7 obtaining a copy of this software and associated documentation
8 files (the "Software"), to deal in the Software without
9 restriction, including without limitation the rights to use,
10 copy, modify, merge, publish, distribute, sublicense, and/or sell
11 copies of the Software, and to permit persons to whom the
12 Software is furnished to do so, subject to the following
13 conditions:
14
15 The above copyright notice and this permission notice shall be
16 included in all copies or substantial portions of the Software.
17
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
20 OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
22 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
23 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
25 OTHER DEALINGS IN THE SOFTWARE.
26 */
27
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
28 /* Simple shared-memory IPC throughput test, with spin-waits rather
29 than blocking. The shared area is divided into packets, each of
30 which starts with a header. The header is the size of a cache
31 line, but the only really interesting field is the size of the
32 packet. To transmit, you just copy the data you want to transmit
33 it and then go back and write the header. Meanwhile, the receiver
34 will be sitting spinning waiting for the header to arrive, and will
35 read the packet out as soon as it does. Once it's done so, it'll
36 go back and set the finished flag in the header, so that the
37 transmitter knows when it's safe to reuse a bit of buffer for a new
38 message. The benchmark which we build on top of this just keeps
39 sending fixed-sized messages for soem number of times, waiting as
40 appropriate for the receiver to pick them up. */
236b14f @avsm add licenses
authored
41
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
42 #include <sys/mman.h>
43 #include <sys/stat.h>
5aecafa Add a simple futex-based version of mempipe. Surprisingly slow.
Steven Smith authored
44 #include <sys/syscall.h>
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
45 #include <sys/time.h>
46 #include <assert.h>
47 #include <err.h>
5aecafa Add a simple futex-based version of mempipe. Surprisingly slow.
Steven Smith authored
48 #include <errno.h>
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
49 #include <fcntl.h>
50 #include <inttypes.h>
51 #include <stdbool.h>
52 #include <stdlib.h>
53 #include <string.h>
54 #include <unistd.h>
79cb197 @avsm do not use futexes at all if USE_FUTEX is not defined, which will cau…
authored
55 #include <sys/uio.h>
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
56
d3204db Lob in some monitor/mwait bits, just to see if they work. Turns out
Steven Smith authored
57 #undef USE_MWAIT
cd68604 @smowton Make a spinny version of mempipe
smowton authored
58 #ifndef NO_FUTEX
5aecafa Add a simple futex-based version of mempipe. Surprisingly slow.
Steven Smith authored
59 #define USE_FUTEX
cd68604 @smowton Make a spinny version of mempipe
smowton authored
60 #endif
d3204db Lob in some monitor/mwait bits, just to see if they work. Turns out
Steven Smith authored
61
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
62 #include "test.h"
63 #include "xutil.h"
79cb197 @avsm do not use futexes at all if USE_FUTEX is not defined, which will cau…
authored
64
65 #ifdef Linux
66 #include <linux/futex.h>
9a3f5ca Split the futex-y operations out into a header file rather than
Steven Smith authored
67 #include "futex.h"
79cb197 @avsm do not use futexes at all if USE_FUTEX is not defined, which will cau…
authored
68 #endif
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
69
70 #define PAGE_SIZE 4096
71 #define CACHE_LINE_SIZE 64
a8e52b0 Make it possible to change the size of the ring in the mempipe
Steven Smith authored
72 static unsigned nr_shared_pages = 512;
73 #define ring_size (PAGE_SIZE * nr_shared_pages)
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
74
75 struct msg_header {
690e015 Stop using the sign of the size field as a flag, and instead use
Steven Smith authored
76 #define MH_FLAG_READY 1
77 #define MH_FLAG_STOP 2
5aecafa Add a simple futex-based version of mempipe. Surprisingly slow.
Steven Smith authored
78 #define MH_FLAG_WAITING 4
79 #define MH_FLAGS (MH_FLAG_READY|MH_FLAG_STOP|MH_FLAG_WAITING)
690e015 Stop using the sign of the size field as a flag, and instead use
Steven Smith authored
80 unsigned size_and_flags;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
81 int pad[CACHE_LINE_SIZE / sizeof(int) - 1];
82 };
83
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
84 struct ring_state {
85 void* ringmem;
86 unsigned long next_tx_offset;
87 unsigned long first_unacked_msg;
88 unsigned long next_message_start;
89 struct iovec vecs[2];
90 };
91
a8e52b0 Make it possible to change the size of the ring in the mempipe
Steven Smith authored
92 static unsigned long
93 mask_ring_index(unsigned long idx)
94 {
95 return idx & (ring_size - 1);
96 }
97
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
98 static void
99 init_test(test_data *td)
100 {
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
101 struct ring_state* rs = (struct ring_state*)xmalloc(sizeof(struct ring_state));
b75a059 Merge branch 'master' of git://github.com/avsm/ipc-bench
Steven Smith authored
102 rs->ringmem = establish_shm_segment(nr_shared_pages, td->numa_node);
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
103 td->data = rs;
9ff1b7b @smowton Add support for optional writing of actual data, plus actual support …
smowton authored
104 }
105
d3204db Lob in some monitor/mwait bits, just to see if they work. Turns out
Steven Smith authored
106 #ifdef USE_MWAIT
107 static void
108 mwait(void)
109 {
110 asm volatile("mwait\n"
111 :
112 : "a" (0), "c" (0));
113 }
114 static void
115 monitor(const volatile void *what)
116 {
117 asm volatile("monitor\n"
118 :
119 : "a" (what),
120 "c" (0),
121 "d" (0)
122 );
123 }
124 #else
125 static void
126 mwait(void)
127 {
128 }
129 static void
130 monitor(const volatile void *what)
131 {
132 }
133 #endif
134
27f19da Try to reduce the number of atomic operations necessary for futex mode
Steven Smith authored
135 /* Deferred write state */
136 #ifdef USE_FUTEX
137 static struct {
138 volatile unsigned *ptr;
139 unsigned val;
140 unsigned cntr;
141 } deferred_write;
142
143 static void
144 _set_message_ready(volatile unsigned *ptr, unsigned val)
145 {
146 int sz;
147 sz = atomic_xchg(ptr, val);
148 if (sz & MH_FLAG_WAITING)
149 futex_wake(ptr);
150 }
151 #endif
152
444e4da Duplicate code reduction.
Steven Smith authored
153 static int
154 wait_for_message_ready(volatile struct msg_header *mh, int desired_state)
155 {
156 int sz;
157 #ifdef USE_FUTEX
158 int new_sz;
27f19da Try to reduce the number of atomic operations necessary for futex mode
Steven Smith authored
159 if (deferred_write.ptr) {
160 _set_message_ready(deferred_write.ptr, deferred_write.val);
161 deferred_write.ptr = NULL;
162 }
444e4da Duplicate code reduction.
Steven Smith authored
163 while (1) {
164 sz = mh->size_and_flags;
165 if ((sz & MH_FLAG_READY) == desired_state)
166 break;
167 new_sz = sz | MH_FLAG_WAITING;
168 if (new_sz == sz ||
169 atomic_cmpxchg(&mh->size_and_flags, sz, new_sz) == sz)
170 futex_wait_while_equal(&mh->size_and_flags, new_sz);
171 }
172 #else
173 sz = mh->size_and_flags;
174 if ((sz & MH_FLAG_READY) != desired_state) {
175 while (1) {
176 monitor(&mh->size_and_flags);
177 sz = mh->size_and_flags;
178 if ((sz & MH_FLAG_READY) == desired_state)
179 break;
180 mwait();
181 }
182 }
183 #endif
184 return sz;
185 }
186
187 static void
188 set_message_ready(volatile struct msg_header *mh, int size)
189 {
190 #ifdef USE_FUTEX
27f19da Try to reduce the number of atomic operations necessary for futex mode
Steven Smith authored
191 if (deferred_write.ptr) {
192 deferred_write.cntr--;
193 mh->size_and_flags = size;
194 if (!deferred_write.cntr) {
195 _set_message_ready(deferred_write.ptr, deferred_write.val);
196 deferred_write.ptr = NULL;
197 }
198 } else {
199 deferred_write.ptr = &mh->size_and_flags;
200 deferred_write.val = size;
201 deferred_write.cntr = 2;
202 }
444e4da Duplicate code reduction.
Steven Smith authored
203 #else
204 mh->size_and_flags = size;
205 #endif
206 }
d3204db Lob in some monitor/mwait bits, just to see if they work. Turns out
Steven Smith authored
207
9ff1b7b @smowton Add support for optional writing of actual data, plus actual support …
smowton authored
208 static void
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
209 init_child(test_data *td)
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
210 {
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
211 struct ring_state* rs = (struct ring_state*)td->data;
212 volatile struct msg_header *mh = rs->ringmem;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
213
214 /* Sync up with parent */
690e015 Stop using the sign of the size field as a flag, and instead use
Steven Smith authored
215 mh->size_and_flags = 0xf008;
216 while (mh->size_and_flags == 0xf008)
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
217 ;
218
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
219 rs->next_message_start = 0;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
220 /* Enter main message loop */
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
221 }
5aecafa Add a simple futex-based version of mempipe. Surprisingly slow.
Steven Smith authored
222
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
223 static struct iovec* get_read_buffer(test_data* td, int len, int* n_vecs) {
5aecafa Add a simple futex-based version of mempipe. Surprisingly slow.
Steven Smith authored
224
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
225 struct ring_state* rs = (struct ring_state*)td->data;
226 volatile struct msg_header *mh;
227 int sz;
228 assert(rs->next_message_start % CACHE_LINE_SIZE == 0);
229 mh = rs->ringmem + mask_ring_index(rs->next_message_start);
230 sz = wait_for_message_ready(mh, MH_FLAG_READY);
231 if (sz & MH_FLAG_STOP) { /* End of test */
232 *n_vecs = 0;
233 return 0;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
234 }
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
235 sz &= ~MH_FLAGS;
236 if(sz != td->size) {
237 exit(1);
238 }
239 unsigned long offset = mask_ring_index(rs->next_message_start + sizeof(struct msg_header));
240 if (offset + td->size <= ring_size) {
241 rs->vecs[0].iov_base = rs->ringmem + offset;
242 rs->vecs[0].iov_len = td->size;
243 *n_vecs = 1;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
244 }
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
245 else {
246 rs->vecs[0].iov_base = rs->ringmem + offset;
247 rs->vecs[0].iov_len = ring_size - offset;
248 rs->vecs[1].iov_base = rs->ringmem;
249 rs->vecs[1].iov_len = td->size - (ring_size - offset);
250 *n_vecs = 2;
251 }
252
253 return rs->vecs;
254
255 }
256
257 static void release_read_buffer(test_data* td, struct iovec* vecs, int n_vecs) {
258
259 struct ring_state* rs = (struct ring_state*)td->data;
260 volatile struct msg_header *mh = rs->ringmem + mask_ring_index(rs->next_message_start);
261
262 set_message_ready(mh, td->size);
263
264 rs->next_message_start += td->size + sizeof(struct msg_header);
265
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
266 }
267
2e1109a @smowton Debug ported tested and make verify step optional
smowton authored
268 static void child_finish(test_data* td) {
269
270 #ifdef USE_FUTEX
271 if(deferred_write.ptr)
272 _set_message_ready(deferred_write.ptr, deferred_write.val);
273 #endif
274
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
275 }
276
277 static void
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
278 init_parent(test_data *td)
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
279 {
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
280 struct ring_state* rs = (struct ring_state*)td->data;
281 volatile struct msg_header *mh = rs->ringmem;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
282
a8e52b0 Make it possible to change the size of the ring in the mempipe
Steven Smith authored
283 assert(td->size < ring_size - sizeof(struct msg_header));
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
284
285 /* Wait for child to show up. */
690e015 Stop using the sign of the size field as a flag, and instead use
Steven Smith authored
286 while (mh->size_and_flags != 0xf008)
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
287 ;
690e015 Stop using the sign of the size field as a flag, and instead use
Steven Smith authored
288 mh->size_and_flags = 0;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
289
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
290 rs->next_tx_offset = 0;
291 rs->first_unacked_msg = 0;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
292
293 /* Round up to multiple of cache line size, for sanity. */
294 td->size = td->size + CACHE_LINE_SIZE - 1;
295 td->size -= td->size % CACHE_LINE_SIZE;
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
296 }
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
297
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
298 struct iovec*
299 get_write_buffer(test_data* td, int len, int* n_vecs) {
300
301 struct ring_state* rs = (struct ring_state*)td->data;
302 volatile struct msg_header *mh;
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
303
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
304 assert(len == td->size);
305
306 /* Check for available ring space (eom = end of message) */
307 unsigned long eom = rs->next_tx_offset + td->size + sizeof(struct msg_header) * 2;
308 while (eom - rs->first_unacked_msg > ring_size) {
309 int size;
310 mh = rs->ringmem + mask_ring_index(rs->first_unacked_msg);
311 size = wait_for_message_ready(mh, 0);
312 size &= ~MH_FLAGS;
313 rs->first_unacked_msg += size + sizeof(struct msg_header);
314 }
315
316 unsigned long offset = mask_ring_index(rs->next_tx_offset + sizeof(struct msg_header));
317 if (offset + td->size <= ring_size) {
318 rs->vecs[0].iov_base = rs->ringmem + offset;
319 rs->vecs[0].iov_len = td->size;
320 *n_vecs = 1;
321 } else {
322 rs->vecs[0].iov_base = rs->ringmem + offset;
323 rs->vecs[0].iov_len = ring_size - offset;
324 rs->vecs[1].iov_base = rs->ringmem;
325 rs->vecs[1].iov_len = td->size - (ring_size - offset);
326 *n_vecs = 2;
327 }
328
329 return rs->vecs;
330
331 }
332
333 void release_write_buffer(test_data* td, struct iovec* vecs, int nvecs) {
334
335 struct ring_state* rs = (struct ring_state*)td->data;
336 volatile struct msg_header *mh;
337 volatile struct msg_header *mh2;
338 assert(vecs == rs->vecs);
339
340 /* Send message */
341 mh = rs->ringmem + mask_ring_index(rs->next_tx_offset);
342
343 /* Make sure that the size field in the *next* message is clear
344 before setting the size field in *this* message. That makes
345 sure that the receiver stops and spins in the right place,
346 rather than wandering off into la-la land if it picks up a
347 stale message. */
348 mh2 = rs->ringmem + mask_ring_index(rs->next_tx_offset + td->size + sizeof(struct msg_header));
349 mh2->size_and_flags = 0;
350
351 set_message_ready(mh, td->size | MH_FLAG_READY);
352
353 rs->next_tx_offset += td->size + sizeof(struct msg_header);
354
355 }
356
357 void parent_finish(test_data* td) {
358
359 struct ring_state* rs = (struct ring_state*)td->data;
360 volatile struct msg_header *mh;
2e1109a @smowton Debug ported tested and make verify step optional
smowton authored
361
362 mh = rs->ringmem + mask_ring_index(rs->next_tx_offset);
690e015 Stop using the sign of the size field as a flag, and instead use
Steven Smith authored
363 mh->size_and_flags = MH_FLAG_READY | MH_FLAG_STOP;
79cb197 @avsm do not use futexes at all if USE_FUTEX is not defined, which will cau…
authored
364 #ifdef USE_FUTEX
431f1e8 Unbreak mempipe_thr a bit.
Steven Smith authored
365 futex_wake(&mh->size_and_flags);
79cb197 @avsm do not use futexes at all if USE_FUTEX is not defined, which will cau…
authored
366 #endif
2e1109a @smowton Debug ported tested and make verify step optional
smowton authored
367
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
368 /* Wait for child to acknowledge receipt of all messages */
369 while (rs->first_unacked_msg != rs->next_tx_offset) {
370 int size;
371 mh = rs->ringmem + mask_ring_index(rs->first_unacked_msg);
372 size = wait_for_message_ready(mh, 0);
373 size &= ~MH_FLAGS;
374 rs->first_unacked_msg += size + sizeof(struct msg_header);
375 }
376
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
377 }
378
d3204db Lob in some monitor/mwait bits, just to see if they work. Turns out
Steven Smith authored
379 #ifdef USE_MWAIT
380 static void
381 cpuid(int leaf, unsigned long *a, unsigned long *b, unsigned long *c, unsigned long *d)
382 {
383 unsigned long _a, _b, _c, _d;
384 asm ("cpuid"
385 : "=a" (_a), "=b" (_b), "=c" (_c), "=d" (_d)
386 : "0" (leaf)
387 );
388 if (a)
389 *a = _a;
390 if (b)
391 *b = _b;
392 if (c)
393 *c = _c;
394 if (d)
395 *d = _d;
396 }
397
398 static void
399 check_monitor_line_size(void)
400 {
401 unsigned long a, b, c;
402 cpuid(5, &a, &b, NULL, NULL);
403 assert(a == b);
404 assert(a == CACHE_LINE_SIZE);
405 cpuid(1, NULL, NULL, &c, NULL);
406 printf("Available: %d\n", !!(c & (1 << 3)));
407 }
408 #else
409 static void
410 check_monitor_line_size(void)
411 {
412 }
413 #endif
414
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
415 int
416 main(int argc, char *argv[])
417 {
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
418 test_t t = {
d30575b @smowton Name the spinning version of mempipe mempipe_spin_thr
smowton authored
419 .name = "mempipe_"
420 #ifdef NO_FUTEX
421 "spin_"
422 #endif
423 "thr",
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
424 .is_latency_test = 0,
425 .init_test = init_test,
426 .init_parent = init_parent,
427 .finish_parent = parent_finish,
428 .init_child = init_child,
2e1109a @smowton Debug ported tested and make verify step optional
smowton authored
429 .finish_child = child_finish,
ae96c21 @smowton Move the more difficult tests to the improved test harness
smowton authored
430 .get_write_buffer = get_write_buffer,
431 .release_write_buffer = release_write_buffer,
432 .get_read_buffer = get_read_buffer,
433 .release_read_buffer = release_read_buffer
434 };
a8e52b0 Make it possible to change the size of the ring in the mempipe
Steven Smith authored
435 char *ring_order = getenv("MEMPIPE_RING_ORDER");
d3204db Lob in some monitor/mwait bits, just to see if they work. Turns out
Steven Smith authored
436 check_monitor_line_size();
a8e52b0 Make it possible to change the size of the ring in the mempipe
Steven Smith authored
437 if (ring_order) {
438 int as_int;
439 if (sscanf(ring_order, "%d", &as_int) != 1)
440 err(1, "MEMPIPE_RING_ORDER must be an integer");
441 if (as_int < 0 || as_int > 15)
442 errx(1, "MEMPIPE_RING_ORDER must be between 0 and 15");
443 nr_shared_pages = 1 << as_int;
444 }
985fdd0 Simple in-memory IPC throughput test.
Steven Smith authored
445 run_test(argc, argv, &t);
446 return 0;
447 }
Something went wrong with that request. Please try again.