From 98d94c9c1b4e87abd4c2fffe939105621cebf6e2 Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 10 Jan 2020 10:03:21 +0100 Subject: [PATCH 1/6] add native spark check --- cscs-checks/apps/spark/spark_check.py | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 cscs-checks/apps/spark/spark_check.py diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py new file mode 100644 index 0000000000..909e37d50c --- /dev/null +++ b/cscs-checks/apps/spark/spark_check.py @@ -0,0 +1,46 @@ +import math + +import reframe as rfm +import reframe.utility.sanity as sn +from reframe.core.launchers.registry import getlauncher + + +@rfm.simple_test +class SparkAnalyticsCheck(rfm.RunOnlyRegressionTest): + def __init__(self): + super().__init__() + self.descr = 'Simple calculation of pi with Spark' + self.valid_systems = ['daint:gpu', 'daint:mc', + 'dom:gpu', 'dom:mc'] + self.valid_prog_environs = ['PrgEnv-cray'] + self.modules = ['Spark'] + self.sourcesdir = None + self.variables = { + 'SPARK_WORKER_CORES': '36', + 'SPARK_LOCAL_DIRS': '"/tmp"', + } + # `SPARK_CONF` needs to be defined after running `start-all.sh`. + self.pre_run = [ + 'start-all.sh', + ('SPARK_CONF="--conf spark.default.parallelism=10 ' + '--conf spark.executor.cores=8 ' + '--conf spark.executor.memory=15g"') + ] + self.executable = ( + 'spark-submit ${SPARK_CONF} --master $SPARKURL ' + '--class org.apache.spark.examples.SparkPi ' + '$EBROOTSPARK/examples/jars/spark-examples_2.11-2.3.1.jar 10000;') + self.post_run = ['stop-all.sh'] + self.num_tasks = 2 + self.num_tasks_per_node = 1 + pi_value = sn.extractsingle(r'Pi is roughly\s+(?P\S+)', + self.stdout, 'pi', float) + self.sanity_patterns = sn.assert_lt(sn.abs(pi_value - math.pi), 0.01) + self.maintainers = ['TM', 'TR'] + self.tags = {'production'} + + def setup(self, partition, environ, **job_opts): + super().setup(partition, environ, **job_opts) + # The job launcher has to be changed since the `start_analytics` + # script is not used with srun. + self.job.launcher = getlauncher('local')() From ce0f5a5c77588cae14dab6a1a22c394fadd720af Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 10 Jan 2020 10:10:28 +0100 Subject: [PATCH 2/6] fix check class name --- cscs-checks/apps/spark/spark_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py index 909e37d50c..5389bfe876 100644 --- a/cscs-checks/apps/spark/spark_check.py +++ b/cscs-checks/apps/spark/spark_check.py @@ -6,7 +6,7 @@ @rfm.simple_test -class SparkAnalyticsCheck(rfm.RunOnlyRegressionTest): +class SparkCheck(rfm.RunOnlyRegressionTest): def __init__(self): super().__init__() self.descr = 'Simple calculation of pi with Spark' From b7672e73c4516d5dbfc661d33fec0305bc30ad6c Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 10 Jan 2020 11:13:14 +0100 Subject: [PATCH 3/6] fix comments --- cscs-checks/apps/spark/spark_check.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py index 5389bfe876..336edfe09a 100644 --- a/cscs-checks/apps/spark/spark_check.py +++ b/cscs-checks/apps/spark/spark_check.py @@ -12,13 +12,9 @@ def __init__(self): self.descr = 'Simple calculation of pi with Spark' self.valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc'] - self.valid_prog_environs = ['PrgEnv-cray'] + self.valid_prog_environs = ['PrgEnv-gnu'] self.modules = ['Spark'] self.sourcesdir = None - self.variables = { - 'SPARK_WORKER_CORES': '36', - 'SPARK_LOCAL_DIRS': '"/tmp"', - } # `SPARK_CONF` needs to be defined after running `start-all.sh`. self.pre_run = [ 'start-all.sh', @@ -41,6 +37,15 @@ def __init__(self): def setup(self, partition, environ, **job_opts): super().setup(partition, environ, **job_opts) + if partition.name == 'gpu': + num_workers = 12 + else: + num_workers = 36 + + self.variables = { + 'SPARK_WORKER_CORES': '%s' % num_workers, + 'SPARK_LOCAL_DIRS': '"/tmp"', + } # The job launcher has to be changed since the `start_analytics` # script is not used with srun. self.job.launcher = getlauncher('local')() From 4ed220533db96e52aff9f6a920d832120a369975 Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 10 Jan 2020 11:18:27 +0100 Subject: [PATCH 4/6] fix comments --- cscs-checks/apps/spark/spark_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py index 336edfe09a..ffa1c57e34 100644 --- a/cscs-checks/apps/spark/spark_check.py +++ b/cscs-checks/apps/spark/spark_check.py @@ -46,6 +46,6 @@ def setup(self, partition, environ, **job_opts): 'SPARK_WORKER_CORES': '%s' % num_workers, 'SPARK_LOCAL_DIRS': '"/tmp"', } - # The job launcher has to be changed since the `start_analytics` + # The job launcher has to be changed since the `spark-submit` # script is not used with srun. self.job.launcher = getlauncher('local')() From 9fa39f7dce0c656950cc92da34d84542c0bb2846 Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 10 Jan 2020 11:28:39 +0100 Subject: [PATCH 5/6] fix comments --- cscs-checks/apps/spark/spark_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py index ffa1c57e34..cd8b84a6cd 100644 --- a/cscs-checks/apps/spark/spark_check.py +++ b/cscs-checks/apps/spark/spark_check.py @@ -8,7 +8,6 @@ @rfm.simple_test class SparkCheck(rfm.RunOnlyRegressionTest): def __init__(self): - super().__init__() self.descr = 'Simple calculation of pi with Spark' self.valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc'] From a42fc5792739099906e40702b14ae61f97fa227a Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 10 Jan 2020 15:37:38 +0100 Subject: [PATCH 6/6] fix comments --- cscs-checks/apps/spark/spark_check.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cscs-checks/apps/spark/spark_check.py b/cscs-checks/apps/spark/spark_check.py index cd8b84a6cd..c87b4c53d7 100644 --- a/cscs-checks/apps/spark/spark_check.py +++ b/cscs-checks/apps/spark/spark_check.py @@ -14,17 +14,7 @@ def __init__(self): self.valid_prog_environs = ['PrgEnv-gnu'] self.modules = ['Spark'] self.sourcesdir = None - # `SPARK_CONF` needs to be defined after running `start-all.sh`. - self.pre_run = [ - 'start-all.sh', - ('SPARK_CONF="--conf spark.default.parallelism=10 ' - '--conf spark.executor.cores=8 ' - '--conf spark.executor.memory=15g"') - ] - self.executable = ( - 'spark-submit ${SPARK_CONF} --master $SPARKURL ' - '--class org.apache.spark.examples.SparkPi ' - '$EBROOTSPARK/examples/jars/spark-examples_2.11-2.3.1.jar 10000;') + self.pre_run = ['start-all.sh'] self.post_run = ['stop-all.sh'] self.num_tasks = 2 self.num_tasks_per_node = 1 @@ -34,17 +24,25 @@ def __init__(self): self.maintainers = ['TM', 'TR'] self.tags = {'production'} - def setup(self, partition, environ, **job_opts): - super().setup(partition, environ, **job_opts) - if partition.name == 'gpu': + @rfm.run_before('run') + def prepare_run(self): + if self.current_partition.fullname in ['daint:gpu', 'dom:gpu']: num_workers = 12 + exec_cores = 3 else: num_workers = 36 + exec_cores = 9 self.variables = { 'SPARK_WORKER_CORES': '%s' % num_workers, 'SPARK_LOCAL_DIRS': '"/tmp"', } + self.executable = ( + 'spark-submit --conf spark.default.parallelism=%s ' + '--conf spark.executor.cores=%s --conf spark.executor.memory=15g ' + '--master $SPARKURL --class org.apache.spark.examples.SparkPi ' + '$EBROOTSPARK/examples/jars/spark-examples_2.11-2.3.1.jar 10000;' + % (num_workers, exec_cores)) # The job launcher has to be changed since the `spark-submit` # script is not used with srun. self.job.launcher = getlauncher('local')()