diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py new file mode 100644 index 0000000000..c87b4c53d7 --- /dev/null +++ b/cscs-checks/apps/spark/spark_check.py @@ -0,0 +1,48 @@ +import math + +import reframe as rfm +import reframe.utility.sanity as sn +from reframe.core.launchers.registry import getlauncher + + +@rfm.simple_test +class SparkCheck(rfm.RunOnlyRegressionTest): + def __init__(self): + self.descr = 'Simple calculation of pi with Spark' + self.valid_systems = ['daint:gpu', 'daint:mc', + 'dom:gpu', 'dom:mc'] + self.valid_prog_environs = ['PrgEnv-gnu'] + self.modules = ['Spark'] + self.sourcesdir = None + self.pre_run = ['start-all.sh'] + self.post_run = ['stop-all.sh'] + self.num_tasks = 2 + self.num_tasks_per_node = 1 + pi_value = sn.extractsingle(r'Pi is roughly\s+(?P\S+)', + self.stdout, 'pi', float) + self.sanity_patterns = sn.assert_lt(sn.abs(pi_value - math.pi), 0.01) + self.maintainers = ['TM', 'TR'] + self.tags = {'production'} + + @rfm.run_before('run') + def prepare_run(self): + if self.current_partition.fullname in ['daint:gpu', 'dom:gpu']: + num_workers = 12 + exec_cores = 3 + else: + num_workers = 36 + exec_cores = 9 + + self.variables = { + 'SPARK_WORKER_CORES': '%s' % num_workers, + 'SPARK_LOCAL_DIRS': '"/tmp"', + } + self.executable = ( + 'spark-submit --conf spark.default.parallelism=%s ' + '--conf spark.executor.cores=%s --conf spark.executor.memory=15g ' + '--master $SPARKURL --class org.apache.spark.examples.SparkPi ' + '$EBROOTSPARK/examples/jars/spark-examples_2.11-2.3.1.jar 10000;' + % (num_workers, exec_cores)) + # The job launcher has to be changed since the `spark-submit` + # script is not used with srun. + self.job.launcher = getlauncher('local')()