Skip to content

[test] Add MPI halo cell exchange benchmark #666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 10, 2019
Merged

[test] Add MPI halo cell exchange benchmark #666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
65fef78
benchmark halo cell exchange
ajocksch Feb 1, 2019
cefc720
shortened lines
ajocksch Feb 1, 2019
6aa83fb
bug in halo_cell_exchange fixed
ajocksch Feb 14, 2019
f8e3bef
almost all required changes in
ajocksch Mar 3, 2019
3379e45
implemented changes required
ajocksch Mar 4, 2019
6a75df3
Fix C coding style
Mar 10, 2019
eb5d1c6
Merge branch 'master' into benchmarks/halo_cell_exchange
vkarak Mar 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions cscs-checks/microbenchmarks/mpi/halo_cell_exchange.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import reframe as rfm
import reframe.utility.sanity as sn


@rfm.required_version('>=2.16-dev0')
@rfm.simple_test
class HaloCellExchangeTest(rfm.RegressionTest):
def __init__(self):
super().__init__()
self.sourcepath = 'halo_cell_exchange.c'
self.build_system = 'SingleSource'
self.build_system.cflags = ['-O2']
self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi',
'PrgEnv-gnu']
self.num_tasks = 6
self.num_tasks_per_node = 1
self.num_gpus_per_node = 0

self.executable_opts = ['input.txt']

self.sanity_patterns = sn.assert_eq(
sn.count(sn.findall(r'halo_cell_exchange', self.stdout)), 9)

self.perf_patterns = {
'time_2_10': sn.extractsingle(
r'halo_cell_exchange 6 2 1 1 10 10 10'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_2_10000': sn.extractsingle(
r'halo_cell_exchange 6 2 1 1 10000 10000 10000'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_2_1000000': sn.extractsingle(
r'halo_cell_exchange 6 2 1 1 1000000 1000000 1000000'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_4_10': sn.extractsingle(
r'halo_cell_exchange 6 2 2 1 10 10 10'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_4_10000': sn.extractsingle(
r'halo_cell_exchange 6 2 2 1 10000 10000 10000'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_4_1000000': sn.extractsingle(
r'halo_cell_exchange 6 2 2 1 1000000 1000000 1000000'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_6_10': sn.extractsingle(
r'halo_cell_exchange 6 3 2 1 10 10 10'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_6_10000': sn.extractsingle(
r'halo_cell_exchange 6 3 2 1 10000 10000 10000'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float),
'time_6_1000000': sn.extractsingle(
r'halo_cell_exchange 6 3 2 1 1000000 1000000 1000000'
r' \S+ (?P<time_mpi>\S+)', self.stdout,
'time_mpi', float)
}

# the kesch values need to be added
self.sys_reference = {
'dom:gpu': {
'time_2_10': (3.925395e-06, None, 0.50, 's'),
'time_2_10000': (9.721279e-06, None, 0.50, 's'),
'time_2_1000000': (4.934530e-04, None, 0.50, 's'),
'time_4_10': (5.878997e-06, None, 0.50, 's'),
'time_4_10000': (1.495080e-05, None, 0.50, 's'),
'time_4_1000000': (6.791397e-04, None, 0.50, 's'),
'time_6_10': (5.428815e-06, None, 0.50, 's'),
'time_6_10000': (1.540580e-05, None, 0.50, 's'),
'time_6_1000000': (9.179296e-04, None, 0.50, 's')
},
'daint:gpu': {
'time_2_10': (3.925395e-06, None, 0.50, 's'),
'time_2_10000': (9.721279e-06, None, 0.50, 's'),
'time_2_1000000': (4.934530e-04, None, 0.50, 's'),
'time_4_10': (5.878997e-06, None, 0.50, 's'),
'time_4_10000': (1.495080e-05, None, 0.50, 's'),
'time_4_1000000': (6.791397e-04, None, 0.50, 's'),
'time_6_10': (5.428815e-06, None, 0.50, 's'),
'time_6_10000': (1.540580e-05, None, 0.50, 's'),
'time_6_1000000': (9.179296e-04, None, 0.50, 's')
},
'kesch:cn': {
'time_2_10': (2.280450e-06, None, 0.50, 's'),
'time_2_10000': (8.059907e-06, None, 0.50, 's'),
'time_2_1000000': (5.959686e-04, None, 0.50, 's'),
'time_4_10': (2.951527e-06, None, 0.50, 's'),
'time_4_10000': (1.258132e-05, None, 0.50, 's'),
'time_4_1000000': (8.539153e-04, None, 0.50, 's'),
'time_6_10': (3.740311e-06, None, 0.50, 's'),
'time_6_10000': (1.448979e-05, None, 0.50, 's'),
'time_6_1000000': (8.432294e-04, None, 0.50, 's')
}
}

self.reference = self.sys_reference

self.maintainers = ['AJ']
self.tags = {'benchmark'}
285 changes: 285 additions & 0 deletions cscs-checks/microbenchmarks/mpi/src/halo_cell_exchange.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,285 @@
/* This benchmark emulates a halo cell exchange in n dimensions. The pure
communication is considered without any stencil computation.*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <mpi.h>

#define NCALLS 10000

int main(int argc, const char *argv[])
{
MPI_Comm cart_comm, red_comm;
MPI_Request *request;
MPI_Status *status;
FILE *pFile;
int ndims, reorder, color, end;
int *dim_size, *periods, *halosize;
int comm_size, comm_rank, comm_size_cart, comm_rank_cart;
char *sendbuf, *recvbuf, inputbuf[1000], *pinputbuf;
double start, stop, deltatmin, deltatmax, elapsed_time;
int rank_source, rank_dest, i, j;

if (MPI_Init(NULL, NULL) != 0) {
fprintf(stderr, "MPI_Init() failed\n");
exit(1);
}
if (MPI_Comm_size(MPI_COMM_WORLD, &comm_size) != 0) {
fprintf(stderr, "MPI_Comm_size() failed\n");
exit(1);
}
if (MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank) != 0) {
fprintf(stderr, "MPI_Comm_rank() failed\n");
exit(1);
}
if (argc == 1) {
if (comm_rank == 0) {
printf("%s inputfile\n", argv[0]);
printf("ndims dim1 dim2 ... halosize1 halosize2 ...\n");
}
exit(0);
}
if (comm_rank == 0) {
if (strcmp(argv[1], "-") == 0) {
pFile = stdin;
} else {
pFile = fopen(argv[1], "r");
}
}
end = 0;
while (end == 0) {
if (comm_rank == 0) {
/* read parameters for every single benchmark line by line */
end = (fgets(inputbuf, sizeof(inputbuf) - 1, pFile) == NULL);
if (end == 0) {
pinputbuf = inputbuf;
while (*pinputbuf == ' ')
pinputbuf++;
end = !((*pinputbuf >= '0')
&& (*pinputbuf <= '9'));
pinputbuf = inputbuf;
Copy link
Contributor

@vkarak vkarak Feb 27, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a bit strange the way you are reading the input. Can you explain what you expect as input?

Copy link
Contributor

@vkarak vkarak Feb 27, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why don't you simply do an fscanf() here?

}
}
if (MPI_Bcast(&end, 1, MPI_INT, 0, MPI_COMM_WORLD) != 0) {
fprintf(stderr, "MPI_Bcast() failed\n");
exit(1);
}
if (end == 0) {
if (comm_rank == 0) {
/* read number of dimensions */
sscanf(pinputbuf, "%d", &ndims);
}
if (MPI_Bcast(&ndims, 1, MPI_INT, 0, MPI_COMM_WORLD) != 0) {
fprintf(stderr, "MPI_Bcast() failed\n");
exit(1);
}
dim_size = (int *)malloc(ndims * sizeof(*dim_size));
periods = (int *)malloc(ndims * sizeof(*periods));
halosize = (int *)malloc(ndims * sizeof(*halosize));

for (i = 0; i < ndims; i++) {
periods[i] = 1;
}
reorder = 1;

j = 1;
for (i = 0; i < ndims; i++) {
if (comm_rank == 0) {
while (*pinputbuf == ' ')
pinputbuf++;
while (*pinputbuf != ' ')
pinputbuf++;
while (*pinputbuf == ' ')
pinputbuf++;
/* read number of ranks in every dimension */
sscanf(pinputbuf, "%d", &dim_size[i]);
}
if (MPI_Bcast(&dim_size[i], 1, MPI_INT, 0, MPI_COMM_WORLD) != 0) {
fprintf(stderr, "MPI_Bcast() failed\n");
exit(1);
}
j *= dim_size[i];
}
for (i = 0; i < ndims; i++) {
if (comm_rank == 0) {
while (*pinputbuf == ' ')
pinputbuf++;
while (*pinputbuf != ' ')
pinputbuf++;
while (*pinputbuf == ' ')
pinputbuf++;
/* read halo cell size to be communicated in every
dimension */
sscanf(pinputbuf, "%d", &halosize[i]);
}
if (MPI_Bcast(&halosize[i], 1, MPI_INT, 0, MPI_COMM_WORLD) != 0) {
fprintf(stderr, "MPI_Bcast() failed\n");
exit(1);
}
}
if (j > comm_size) {
if (comm_rank == 0) {
printf("Please run with at least %d MPI ranks.\n", j);
}
if (MPI_Finalize() != 0) {
fprintf(stderr, "MPI_Finalize() failed\n");
exit(1);
}
exit(0);
}

/* use only the number of ranks required */
color = (comm_rank < j);
if (color == 0) {
if (MPI_Comm_split
(MPI_COMM_WORLD, MPI_UNDEFINED, comm_rank,
&red_comm) != 0) {
fprintf(stderr, "MPI_Comm_split() failed\n");
exit(1);
}
} else {
if (MPI_Comm_split(MPI_COMM_WORLD, color, comm_rank, &red_comm)
!= 0) {
fprintf(stderr, "MPI_Comm_split() failed\n");
exit(1);
}
/* cartesian grid communicator */
if (MPI_Cart_create
(red_comm, ndims, dim_size, periods,
reorder, &cart_comm) != 0) {
fprintf(stderr, "MPI_Comm_create() failed\n");
exit(1);
}
if (MPI_Comm_size(cart_comm, &comm_size_cart) != 0) {
fprintf(stderr, "MPI_Comm_size() failed\n");
exit(1);
}
if (MPI_Comm_rank(cart_comm, &comm_rank_cart) != 0) {
fprintf(stderr, "MPI_Comm_rank() failed\n");
exit(1);
}

j = 0;
for (i = 0; i < ndims; i++) {
if (halosize[i] > j) {
j = halosize[i];
}
}
sendbuf = (char *)malloc(ndims * 2 * j * sizeof(char));
recvbuf = (char *)malloc(ndims * 2 * j * sizeof(char));
request =
(MPI_Request *) malloc(ndims * 2 * 2 * sizeof(MPI_Request));
status =
(MPI_Status *) malloc(ndims * 2 * 2 * sizeof(MPI_Status));

start = MPI_Wtime();

for (j = 0; j < NCALLS; j++) {
for (i = 0; i < ndims; i++) {
/* receive data in every direction */
if (MPI_Cart_shift
(cart_comm, i, 1, &rank_source, &rank_dest) != 0) {
fprintf(stderr, "MPI_Cart_shift() failed\n");
exit(1);
}
if (MPI_Irecv
(recvbuf +
i * 2 * halosize[i] *
sizeof(char), halosize[i],
MPI_CHAR, rank_source, 1,
cart_comm, request + i * 2) != 0) {
fprintf(stderr, "MPI_Irecv() failed\n");
exit(1);
}
if (MPI_Irecv
(recvbuf +
(i * 2 +
1) * halosize[i] *
sizeof(char), halosize[i],
MPI_CHAR, rank_dest, 1,
cart_comm, request + i * 2 + 1) != 0) {
fprintf(stderr, "MPI_Irecv() failed\n");
exit(1);
}
}
for (i = 0; i < ndims; i++) {
/* send data in every direction */
if (MPI_Cart_shift
(cart_comm, i, 1, &rank_source, &rank_dest) != 0) {
fprintf(stderr, "MPI_Cart_shift() failed\n");
exit(1);
}
if (MPI_Isend
(sendbuf +
i * 2 * halosize[i] *
sizeof(char), halosize[i],
MPI_CHAR, rank_source, 1,
cart_comm, request + i * 2 + ndims * 2) != 0) {
fprintf(stderr, "MPI_Irecv() failed\n");
exit(1);
}
if (MPI_Isend
(sendbuf +
(i * 2 +
1) * halosize[i] *
sizeof(char), halosize[i],
MPI_CHAR, rank_dest, 1,
cart_comm, request + i * 2 + 1 + ndims * 2) != 0) {
fprintf(stderr, "MPI_Irecv() failed\n");
exit(1);
}
}
if (MPI_Waitall(ndims * 2 * 2, request, status) != 0) {
fprintf(stderr, "MPI_Waitall() failed\n");
exit(1);
}
}

stop = MPI_Wtime();
elapsed_time = stop - start;
if (MPI_Reduce
(&elapsed_time, &deltatmin, 1, MPI_DOUBLE,
MPI_MIN, 0, cart_comm) != 0) {
fprintf(stderr, "MPI_Reduce() failed\n");
exit(1);
}
if (MPI_Reduce
(&elapsed_time, &deltatmax, 1, MPI_DOUBLE,
MPI_MAX, 0, cart_comm) != 0) {
fprintf(stderr, "MPI_Reduce() failed\n");
exit(1);
}
if (comm_rank_cart == 0) {
printf("halo_cell_exchange %d", comm_size);
for (i = 0; i < ndims; i++) {
printf(" %d", dim_size[i]);
}
for (i = 0; i < ndims; i++) {
printf(" %d", halosize[i]);
}
/* print minimum and maximum time per exchange and test */
printf(" %e %e\n", deltatmin / NCALLS, deltatmax / NCALLS);
}
free(status);
free(request);
free(recvbuf);
free(sendbuf);
if (MPI_Comm_free(&cart_comm) != 0) {
fprintf(stderr, "MPI_Comm_free() failed\n");
exit(1);
}
if (MPI_Comm_free(&red_comm) != 0) {
fprintf(stderr, "MPI_Comm_free() failed\n");
exit(1);
}
}
free(halosize);
free(periods);
free(dim_size);
}
}
if (MPI_Finalize() != 0) {
fprintf(stderr, "MPI_Finalize() failed\n");
exit(1);
}
}
Loading