Skip to content

Commit

Permalink
grid: Assert that max number of threads
Browse files Browse the repository at this point in the history
  • Loading branch information
oschuett committed Mar 17, 2021
1 parent 1f5bbc4 commit 38be7c2
Showing 1 changed file with 13 additions and 7 deletions.
20 changes: 13 additions & 7 deletions src/grid/common/grid_library.c
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

static grid_library_globals **per_thread_globals = NULL;
static bool library_initialized = false;
static int max_threads = 0;
static grid_library_config config = {.backend = GRID_BACKEND_AUTO,
.device_id = 0,
.validate = false,
Expand All @@ -37,11 +38,12 @@ void grid_library_init(void) {
abort();
}

const int n = omp_get_max_threads();
per_thread_globals = malloc(n * sizeof(grid_library_globals *));
max_threads = omp_get_max_threads();
per_thread_globals = malloc(max_threads * sizeof(grid_library_globals *));

// Using parallel regions to ensure memory is allocated near a thread's core.
#pragma omp parallel default(none) shared(per_thread_globals) num_threads(n)
#pragma omp parallel default(none) shared(per_thread_globals) \
num_threads(max_threads)
{
const int ithread = omp_get_thread_num();
per_thread_globals[ithread] = malloc(sizeof(grid_library_globals));
Expand All @@ -61,7 +63,7 @@ void grid_library_finalize(void) {
abort();
}

for (int i = 0; i < omp_get_max_threads(); i++) {
for (int i = 0; i < max_threads; i++) {
grid_sphere_cache_free(&per_thread_globals[i]->sphere_cache);
free(per_thread_globals[i]);
}
Expand All @@ -75,7 +77,9 @@ void grid_library_finalize(void) {
* \author Ole Schuett
******************************************************************************/
grid_sphere_cache *grid_library_get_sphere_cache(void) {
return &per_thread_globals[omp_get_thread_num()]->sphere_cache;
const int ithread = omp_get_thread_num();
assert(ithread < max_threads);
return &per_thread_globals[ithread]->sphere_cache;
}

/*******************************************************************************
Expand Down Expand Up @@ -106,7 +110,9 @@ void grid_library_counter_add(const int lp, const enum grid_backend backend,
const int increment) {
const int back = backend - GRID_BACKEND_REF;
const int idx = back * 4 * 20 + kernel * 20 + imin(lp, 19);
per_thread_globals[omp_get_thread_num()]->counters[idx] += increment;
const int ithread = omp_get_thread_num();
assert(ithread < max_threads);
per_thread_globals[ithread]->counters[idx] += increment;
}

/*******************************************************************************
Expand Down Expand Up @@ -135,7 +141,7 @@ void grid_library_print_stats(void (*mpi_sum_func)(long *, int),
double total = 0.0;
for (int i = 0; i < 320; i++) {
counters[i][1] = i;
for (int j = 0; j < omp_get_max_threads(); j++) {
for (int j = 0; j < max_threads; j++) {
counters[i][0] += per_thread_globals[j]->counters[i];
}
mpi_sum_func(&counters[i][0], mpi_comm);
Expand Down

0 comments on commit 38be7c2

Please sign in to comment.