diff --git a/cscs-checks/system/slurm/slurm.py b/cscs-checks/system/slurm/slurm.py index adb36f24e9..dbf99dbce7 100644 --- a/cscs-checks/system/slurm/slurm.py +++ b/cscs-checks/system/slurm/slurm.py @@ -186,3 +186,68 @@ def __init__(self): @rfm.run_before('run') def set_memory_limit(self): self.job.options += ['--mem=2000'] + + +@rfm.simple_test +class MemoryOverconsumptionMpiCheck(SlurmCompiledBaseCheck): + def __init__(self): + super().__init__() + self.maintainers = ['JG'] + self.valid_systems += ['eiger:mc', 'pilatus:mc'] + self.time_limit = '5m' + self.sourcepath = 'eatmemory_mpi.c' + self.tags.add('mem') + self.executable_opts = ['100%'] + self.sanity_patterns = sn.assert_found(r'(oom-kill)|(Killed)', + self.stderr) + # {{{ perf + regex = (r'^Eating \d+ MB\/mpi \*\d+mpi = -\d+ MB memory from \/proc\/' + r'meminfo: total: \d+ GB, free: \d+ GB, avail: \d+ GB, using:' + r' (\d+) GB') + self.perf_patterns = { + 'max_cn_memory': sn.getattr(self, 'reference_meminfo'), + 'max_allocated_memory': sn.max( + sn.extractall(regex, self.stdout, 1, int) + ), + } + no_limit = (0, None, None, 'GB') + self.reference = { + '*': { + 'max_cn_memory': no_limit, + 'max_allocated_memory': ( + sn.getattr(self, 'reference_meminfo'), -0.05, None, 'GB' + ), + } + } + # }}} + + # {{{ hooks + @rfm.run_before('run') + def set_tasks(self): + tasks_per_node = { + 'dom:mc': 36, + 'daint:mc': 36, + 'dom:gpu': 12, + 'daint:gpu': 12, + 'eiger:mc': 128, + 'pilatus:mc': 128, + } + partname = self.current_partition.fullname + self.num_tasks_per_node = tasks_per_node[partname] + self.num_tasks = self.num_tasks_per_node + self.job.launcher.options = ['-u'] + # }}} + + @property + @sn.sanity_function + def reference_meminfo(self): + reference_meminfo = { + 'dom:gpu': 62, + 'dom:mc': 62, + 'daint:gpu': 62, + 'daint:mc': 62, # this will pass with 64 GB and above memory sizes + # this will pass with 256 GB and above memory sizes: + 'eiger:mc': 250, + 'pilatus:mc': 250, + } + return reference_meminfo[self.current_partition.fullname] diff --git a/cscs-checks/system/slurm/src/eatmemory_mpi.c b/cscs-checks/system/slurm/src/eatmemory_mpi.c new file mode 100644 index 0000000000..84be87c8c2 --- /dev/null +++ b/cscs-checks/system/slurm/src/eatmemory_mpi.c @@ -0,0 +1,178 @@ +// MPI version of eatmemory.c from Julio Viera +// 12/2020: add cscs_read_proc_meminfo from jg (cscs) +#include +#include +#include +#include +#include +#include +#include +#include + +#define PROC_FILE "/proc/meminfo" +#define MEMTOTAL 0 +#define MEMFREE 1 +#define MEMCACHED 2 +#define SWAPTOTAL 3 +#define SWAPFREE 4 +#define SWAPCACHED 5 +#define MEMAVAIL 6 +#define MEMORY_PERCENTAGE + +typedef struct { + char *str; + uint32_t val; +} meminfo_t; + +int cscs_read_proc_meminfo(int); + +#ifdef MEMORY_PERCENTAGE +size_t getTotalSystemMemory() { + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGE_SIZE); + return pages * page_size; +} + +size_t getFreeSystemMemory() { + long pages = sysconf(_SC_AVPHYS_PAGES); + long page_size = sysconf(_SC_PAGE_SIZE); + return pages * page_size; +} +#endif + +bool eat(long total, int chunk) { + long i; + int rank, mpi_size; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + for (i = 0; i < total; i += chunk) { + if (rank == 0) { + int mb_mpi = chunk / 1048576; + printf("Eating %d MB/mpi *%dmpi = -%d MB ", mb_mpi, mpi_size, + mb_mpi * mpi_size); + cscs_read_proc_meminfo(i); + } + short *buffer = malloc(sizeof(char) * chunk); + if (buffer == NULL) { + return false; + } + memset(buffer, 0, chunk); + } + return true; +} + +int cscs_read_proc_meminfo(int i) { + FILE *fp; + meminfo_t meminfo[] = {{"MemTotal:", 0}, {"MemFree:", 0}, + {"Cached:", 0}, {"SwapCached:", 0}, + {"SwapTotal:", 0}, {"SwapFree:", 0}, + {"MemAvailable:", 0}, {NULL, 0}}; + fp = fopen(PROC_FILE, "r"); + if (!fp) { + printf("Cannot read %s", PROC_FILE); + return -1; + } + char buf[80]; + while (fgets(buf, sizeof(buf), fp)) { + int i; + for (i = 0; meminfo[i].str; i++) { + size_t len = strlen(meminfo[i].str); + if (!strncmp(buf, meminfo[i].str, len)) { + char *ptr = buf + len + 1; + while (isspace(*ptr)) + ptr++; + sscanf(ptr, "%u kB", &meminfo[i].val); + } + } + } + fclose(fp); + + printf("memory from %s: total: %u GB, free: %u GB, avail: %u GB, using: %u GB\n", + PROC_FILE, + meminfo[MEMTOTAL].val / 1048576, meminfo[MEMFREE].val / 1048576, + meminfo[MEMAVAIL].val / 1048576, + (meminfo[MEMTOTAL].val - meminfo[MEMAVAIL].val) / 1048576); + return 0; +} + +int main(int argc, char *argv[]) { + int rank, mpi_size; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); +#ifdef MEMORY_PERCENTAGE + if (rank == 0) { + printf("memory from sysconf: total: %zd avail: %zd\n", \ + getTotalSystemMemory(), getFreeSystemMemory() ); + } +#endif + int i; + for (i = 0; i < argc; i++) { + char *arg = argv[i]; + if (strcmp(arg, "-h") == 0 || strcmp(arg, "-?") == 0 || argc == 1) { + printf("Usage: eatmemory \n"); + printf("Size can be specified in megabytes or gigabytes in the following " + "way:\n"); + printf("# # Bytes example: 1024\n"); + printf("#M # Megabytes example: 15M\n"); + printf("#G # Gigabytes example: 2G\n"); +#ifdef MEMORY_PERCENTAGE + printf("#%% # Percent example: 50%%\n"); +#endif + printf("\n"); + } else if (i > 0) { + int len = strlen(arg); + char unit = arg[len - 1]; + long size = -1; + int chunk = 33554432; // 32M + // int chunk = 67108864; // 64M + // int chunk = 134217728; // 128M + // int chunk = 268435456; // = 256M + // int chunk=536870912; // = 512M + // int chunk=1073741824; // = 1G + if (!isdigit(unit)) { + if (unit == 'M' || unit == 'G') { + arg[len - 1] = 0; + size = atol(arg) * (unit == 'M' ? 1024 * 1024 : 1024 * 1024 * 1024); + } +#ifdef MEMORY_PERCENTAGE + else if (unit == '%') { + size = (atol(arg) * (long)getFreeSystemMemory()) / 100; + } +#endif + else { + printf("Invalid size format\n"); + exit(0); + } + } else { + size = atoi(arg); + } + + if (rank == 0) { + cscs_read_proc_meminfo(i); + printf("Peak: %d mpi * %ld bytes = %ld Mbytes\n", mpi_size, size, + mpi_size * size / 1000000); + printf("Eating %ld bytes in chunks of %d...\n", size, chunk); + printf("Eating %ld (1byte=8bits) Mbytes in chunks of %d Kbytes\n", + (size / 1000000), (chunk / 1000)); + } + if (eat(size, chunk)) { + if (isatty(fileno(stdin))) { + printf("Done, press any key to free the memory\n"); + } else { + if (rank == 0) + printf("rank %d Done, kill this process to free the memory\n", + rank); + while (true) { + sleep(1); + } + } + } else { + printf("ERROR: Could not allocate the memory"); + } + } + } + + MPI_Finalize(); + return 0; +}