Permalink
Browse files

Add a -n option which allows you to specify which NUMA node shared

memory should be allocate from.

Not terribly well tested at the moment, since my tets box only has a
single node, but it at least gives plausible-looking errors if I try
to select a non-existent one.
  • Loading branch information...
1 parent f572d8f commit e0a16b7d0967e3042ce122974996b4f52a1be167 Steven Smith committed Nov 29, 2011
Showing with 44 additions and 15 deletions.
  1. +2 −2 Makefile
  2. +1 −1 futex_lat.c
  3. +1 −1 mempipe_lat.c
  4. +1 −1 mempipe_thr.c
  5. +1 −1 shmem_pipe_thr.c
  6. +4 −1 test.c
  7. +1 −0 test.h
  8. +30 −6 xutil.c
  9. +3 −2 xutil.h
View
@@ -13,10 +13,10 @@ TARGETS+=summarise_tsc_counters
all: $(TARGETS)
%_lat: atomicio.o test.o xutil.o %_lat.o
- $(CC) -lrt $(CFLAGS) -o $@ $^
+ $(CC) -lrt -lnuma $(CFLAGS) -o $@ $^
%_thr: atomicio.o test.o xutil.o %_thr.o
- $(CC) -lrt $(CFLAGS) -o $@ $^
+ $(CC) -lrt -lnuma $(CFLAGS) -o $@ $^
mempipe_sos22_thr: mempipe_thr.c
$(CC) $(CFLAGS) $^ -c -DSOS22_MEMSET -o $@
View
@@ -124,7 +124,7 @@ doit(void *shm)
int
main(int argc, char *argv[])
{
- void *shm = establish_shm_segment(1);
+ void *shm = establish_shm_segment(1, -1);
int i;
for (i = 0; i < 100; i++)
View
@@ -54,7 +54,7 @@ struct shared_page {
static void
init_test(test_data *td)
{
- td->data = establish_shm_segment(1);
+ td->data = establish_shm_segment(1, td->numa_node);
}
static void
View
@@ -104,7 +104,7 @@ mask_ring_index(unsigned long idx)
static void
init_test(test_data *td)
{
- td->data = establish_shm_segment(nr_shared_pages);
+ td->data = establish_shm_segment(nr_shared_pages, td->numa_node);
}
static void
View
@@ -431,7 +431,7 @@ init_test(test_data *td)
{
struct shmem_pipe *sp = calloc(sizeof(*sp), 1);
int pip[2];
- sp->ring = establish_shm_segment(1 << ring_order);
+ sp->ring = establish_shm_segment(1 << ring_order, td->numa_node);
if (pipe(pip) < 0)
err(1, "pipe()");
sp->child_to_parent_read = pip[0];
View
5 test.c
@@ -79,8 +79,10 @@ run_test(int argc, char *argv[], test_t *test)
int size, parallel;
char *output_dir;
int mode;
+ int numa_node;
- parse_args(argc, argv, &per_iter_timings, &size, &count, &first_cpu, &second_cpu, &parallel, &output_dir, &mode);
+ parse_args(argc, argv, &per_iter_timings, &size, &count, &first_cpu, &second_cpu, &parallel, &output_dir, &mode,
+ &numa_node);
if (mkdir(output_dir, 0755) < 0 && errno != EEXIST)
err(1, "creating directory %s", output_dir);
@@ -96,6 +98,7 @@ run_test(int argc, char *argv[], test_t *test)
td->count = count;
td->per_iter_timings = per_iter_timings;
td->mode = mode;
+ td->numa_node = numa_node;
/* Test-specific init */
test->init_test(td);
pid_t pid2 = fork ();
View
1 test.h
@@ -38,6 +38,7 @@ typedef struct {
const char *output_dir;
const char *name;
int mode;
+ int numa_node;
} test_data;
typedef struct {
View
36 xutil.c
@@ -29,6 +29,7 @@
#include <unistd.h>
#include <fcntl.h>
#include <math.h>
+#include <numa.h>
#include <sched.h>
#include <stdarg.h>
#include <stdbool.h>
@@ -159,7 +160,7 @@ dump_tsc_counters(test_data *td, unsigned long *counts, int nr_samples)
static void
help(char *argv[])
{
- fprintf(stderr, "Usage:\n%s [-h] [-a <cpuid>] [-b <cpuid>] [-p <num] [-t] [-s <bytes>] [-c <num>] [-o <directory>]\n", argv[0]);
+ fprintf(stderr, "Usage:\n%s [-h] [-a <cpuid>] [-b <cpuid>] [-p <num] [-t] [-s <bytes>] [-c <num>] [-o <directory>] [-n <node>]\n", argv[0]);
fprintf(stderr, "-h: show this help message\n");
fprintf(stderr, "-a: CPU id that the first process should have affinity with\n");
fprintf(stderr, "-b: CPU id that the second process should have affinity with\n");
@@ -168,12 +169,13 @@ help(char *argv[])
fprintf(stderr, "-s: Size of each packet\n");
fprintf(stderr, "-c: Number of iterations\n");
fprintf(stderr, "-o: Where to put the various output files\n");
+ fprintf(stderr, "-n: NUMA node for shared arena, if any\n");
exit(1);
}
void
parse_args(int argc, char *argv[], bool *per_iter_timings, int *size, size_t *count, int *first_cpu, int *second_cpu,
- int *parallel, char **output_dir, int *mode)
+ int *parallel, char **output_dir, int *mode, int *numa_node)
{
int opt;
*per_iter_timings = false;
@@ -184,7 +186,8 @@ parse_args(int argc, char *argv[], bool *per_iter_timings, int *size, size_t *co
*count = 100;
*output_dir = "results";
*mode = 0;
- while((opt = getopt(argc, argv, "h?tp:a:b:s:c:o:m:")) != -1) {
+ *numa_node = -1;
+ while((opt = getopt(argc, argv, "h?tp:a:b:s:c:o:m:n:")) != -1) {
switch(opt) {
case 't':
*per_iter_timings = true;
@@ -210,6 +213,9 @@ parse_args(int argc, char *argv[], bool *per_iter_timings, int *size, size_t *co
case 'm':
*mode = atoi(optarg);
break;
+ case 'n':
+ *numa_node = atoi(optarg);
+ break;
case '?':
case 'h':
help(argv);
@@ -218,9 +224,9 @@ parse_args(int argc, char *argv[], bool *per_iter_timings, int *size, size_t *co
help(argv);
}
}
- fprintf(stderr, "size %d count %" PRId64 " first_cpu %d second_cpu %d parallel %d tsc %d output_dir %s\n",
+ fprintf(stderr, "size %d count %" PRId64 " first_cpu %d second_cpu %d parallel %d tsc %d output_dir %s numa_node %d\n",
*size, *count, *first_cpu, *second_cpu, *parallel, *per_iter_timings,
- *output_dir);
+ *output_dir, *numa_node);
}
void
@@ -243,10 +249,22 @@ setaffinity(int cpunum)
}
void *
-establish_shm_segment(int nr_pages)
+establish_shm_segment(int nr_pages, int numa_node)
{
int fd;
void *addr;
+ struct bitmask *alloc_nodes;
+ struct bitmask *old_mask;
+
+ if (numa_node != -1) {
+ old_mask = numa_get_membind();
+ alloc_nodes = numa_allocate_nodemask();
+ numa_bitmask_setbit(alloc_nodes, numa_node);
+ numa_set_membind(alloc_nodes);
+ } else {
+ /* Shut the compiler up */
+ alloc_nodes = old_mask = (struct bitmask *)0xf001ul;
+ }
fd = shm_open("/memflag_lat", O_RDWR|O_CREAT|O_EXCL, 0600);
if (fd < 0)
@@ -260,6 +278,12 @@ establish_shm_segment(int nr_pages)
err(1, "mapping shared memory segment");
close(fd);
+ if (numa_node != -1) {
+ numa_set_membind(old_mask);
+ numa_bitmask_free(old_mask);
+ numa_bitmask_free(alloc_nodes);
+ }
+
return addr;
}
View
@@ -32,7 +32,8 @@ void xwrite(int, const void *, size_t);
void setaffinity(int);
void parse_args(int argc, char *argv[], bool *per_iter_timings, int *size, size_t *count,
- int *first_cpu, int *second_cpu, int *parallel, char **output_dir, int *mode);
-void *establish_shm_segment(int nr_pages);
+ int *first_cpu, int *second_cpu, int *parallel, char **output_dir, int *mode,
+ int *numa_node);
+void *establish_shm_segment(int nr_pages, int numa_node);
#define PAGE_SIZE 4096

0 comments on commit e0a16b7

Please sign in to comment.