MWT-26

scale-tests/configs/2020-05-14-mwt25dr.env

@@ -0,0 +1,168 @@
+# Depends on:
+# - TEST_NAME
+# - TEST_S3_BUCKET
+# - TEST_S3_FOLDER
+
+# Workload configuration #######################################################
+#
+# Total CPU quota: 88
+# Total MEM quota: 200000
+# Total GPU quota: 40
+
+CLUSTER_URL="https://mw25dry.scaletesting.mesosphe.re/"
+SECURITY="strict"
+
+DCOS_CLI_URL="https://downloads.dcos.io/cli/releases/binaries/dcos/linux/x86-64/latest/dcos"
+
+# Test configuration ###########################################################
+
+SHOULD_INSTALL_INFRASTRUCTURE=true
+SHOULD_INSTALL_NON_GPU_DISPATCHERS=true
+SHOULD_INSTALL_GPU_DISPATCHERS=false
+SHOULD_RUN_FINITE_STREAMING_JOBS=true
+SHOULD_RUN_INFINITE_STREAMING_JOBS=true
+SHOULD_RUN_BATCH_JOBS=true
+SHOULD_RUN_GPU_BATCH_JOBS=false
+SHOULD_UNINSTALL_INFRASTRUCTURE_AT_THE_END=false
+
+# Infrastructure configuration #################################################
+
+GROUP_NAME="${TEST_NAME}"
+
+SERVICE_NAMES_PREFIX="${TEST_NAME}/"
+INFRASTRUCTURE_OUTPUT_FILE="infrastructure.json"
+
+KAFKA_CLUSTER_COUNT=1
+CASSANDRA_CLUSTER_COUNT=1
+
+ZOOKEEPER_CPUS=10
+ZOOKEEPER_MEM=20000
+ZOOKEEPER_CONFIG='scale-tests/configs/kafka-zookeeper-options.json'
+# Note: empty package repo values will default to latest Universe packages.
+ZOOKEEPER_PACKAGE_REPO=
+# 2.7.0-5.1.2e from the Universe.
+
+KAFKA_CPUS=10
+KAFKA_MEM=20000
+KAFKA_CONFIG='scale-tests/configs/kafka-options.json'
+# Note: empty package repo values will default to latest Universe packages.
+KAFKA_PACKAGE_REPO=
+# 2.9.0-5.4.0 from the Universe.
+
+CASSANDRA_CPUS=10
+CASSANDRA_MEM=20000
+CASSANDRA_CONFIG='scale-tests/configs/cassandra-options.json'
+# Note: empty package repo values will default to latest Universe packages.
+CASSANDRA_PACKAGE_REPO=
+# 2.9.0-3.11.6 from the Universe.
+
+# DSEngine configuration #######################################################
+
+DSENGINE_CPUS=10
+DSENGINE_MEM=20000
+DSENGINE_GPUS=40
+DSENGINE_PACKAGE_REPO=
+
+# Spark configuration ##########################################################
+
+SPARK_CONFIG='scale-tests/configs/spark-options.json'
+
+# Note: empty package repo values will default to latest Universe packages.
+# Spark version 2.10.0-2.4.5
+SPARK_PACKAGE_REPO=https://infinity-artifacts.s3.amazonaws.com/permanent/spark/2.10.0-2.4.5/stub-universe-spark.json
+
+# Note: leaving the Spark executor Docker image empty so that executors inherit
+# the image used for dispatchers.
+SPARK_EXECUTOR_DOCKER_IMAGE=
+
+# Non-GPU Spark dispatchers configuration ######################################
+
+# Not currently used.
+BATCH_MAX_NON_GPU_JOBS=30
+
+SPARK_NON_GPU_DISPATCHERS=3
+SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE="non-gpu-dispatchers.out"
+# Note: this name is built internally by the deploy-dispatchers.py script.
+SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json"
+# Note: driver resources used per dispatcher (1 dispatcher will be able to run
+# 8 drivers since each driver requires 1 CPU).
+SPARK_NON_GPU_QUOTA_DRIVERS_CPUS=8
+SPARK_NON_GPU_QUOTA_DRIVERS_MEM=20000
+# Note: executor resources used per job (1 driver will run 1 job).
+SPARK_NON_GPU_QUOTA_EXECUTORS_CPUS=8
+SPARK_NON_GPU_QUOTA_EXECUTORS_MEM=20000
+
+# GPU Spark dispatchers configuration ##########################################
+
+# Not currently used.
+BATCH_MAX_GPU_JOBS=2
+
+SPARK_GPU_DISPATCHERS=0
+SPARK_GPU_DISPATCHERS_OUTPUT_FILE="gpu-dispatchers.out"
+SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json" # NOTE: this name is built internally by the deploy-dispatchers.py script.
+SPARK_GPU_QUOTA_DRIVERS_CPUS=
+SPARK_GPU_QUOTA_DRIVERS_MEM=
+SPARK_GPU_QUOTA_DRIVERS_GPUS=
+SPARK_GPU_QUOTA_EXECUTORS_CPUS=
+SPARK_GPU_QUOTA_EXECUTORS_MEM=
+SPARK_GPU_QUOTA_EXECUTORS_GPUS=
+
+# Common streaming jobs configuration ##########################################
+
+TEST_ASSEMBLY_JAR_URL='http://infinity-artifacts.s3.amazonaws.com/scale-tests/dcos-spark-scala-tests-assembly-2.4.0-20190325.jar'
+DISPATCHERS_JSON_OUTPUT_FILE="all-dispatchers.json"
+
+# Finite streaming jobs configuration ##########################################
+
+STREAMING_FINITE_SUBMISSIONS_OUTPUT_FILE="finite-submissions.out"
+STREAMING_FINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 3 dispatchers -> 3 producers.
+STREAMING_FINITE_CONSUMERS_PER_PRODUCER=1 # 3 producers -> 3 consumers.
+# 3 producers + 3 consumers = 6 total finite streaming jobs
+STREAMING_FINITE_PRODUCER_NUMBER_OF_WORDS=7692
+STREAMING_FINITE_PRODUCER_WORDS_PER_SECOND=1
+# 7692 words / 1 word per second -> ~2h runtime.
+STREAMING_FINITE_PRODUCER_SPARK_CORES_MAX=2
+STREAMING_FINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
+STREAMING_FINITE_CONSUMER_BATCH_SIZE_SECONDS=10
+STREAMING_FINITE_CONSUMER_SPARK_CORES_MAX=1
+STREAMING_FINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
+
+# Infinite streaming jobs configuration ########################################
+
+STREAMING_INFINITE_SUBMISSIONS_OUTPUT_FILE="infinite-submissions.out"
+STREAMING_INFINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 3 dispatchers -> 3 producers.
+STREAMING_INFINITE_CONSUMERS_PER_PRODUCER=1 # 3 producers -> 3 consumers.
+# 3 producers + 3 consumers = 6 total infinite streaming jobs
+STREAMING_INFINITE_PRODUCER_NUMBER_OF_WORDS=0
+STREAMING_INFINITE_PRODUCER_WORDS_PER_SECOND=1
+STREAMING_INFINITE_PRODUCER_SPARK_CORES_MAX=2
+STREAMING_INFINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
+STREAMING_INFINITE_CONSUMER_BATCH_SIZE_SECONDS=10
+STREAMING_INFINITE_CONSUMER_SPARK_CORES_MAX=1
+STREAMING_INFINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
+
+# Batch jobs configuration #####################################################
+
+SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
+
+BATCH_APP_ID="/${SERVICE_NAMES_PREFIX}batch-workload"
+BATCH_SCRIPT_CPUS=6
+BATCH_SCRIPT_MEM=12288
+BATCH_SUBMITS_PER_MIN=13
+# TODO: update to master for the next MWT.
+BATCH_SPARK_BUILD_BRANCH="dcos-58437-deploy-workloads-under-role-enforced-group"
+
+# Batch GPU jobs configuration #################################################
+
+SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
+
+GPU_APP_ID="/${SERVICE_NAMES_PREFIX}gpu-batch-workload"
+GPU_SCRIPT_CPUS=2
+GPU_SCRIPT_MEM=4096
+GPU_DOCKER_IMAGE='samvantran/spark-dcos-gpu:metrics'
+GPU_SUBMITS_PER_MIN=5
+GPU_MAX_DISPATCHERS=${SPARK_GPU_DISPATCHERS}
+GPU_SPARK_CORES_MAX=4
+GPU_SPARK_MESOS_EXECUTOR_GPUS=4
+GPU_SPARK_MESOS_MAX_GPUS=4
+GPU_SPARK_BUILD_BRANCH=master

scale-tests/configs/2020-05-20-mwt25.env

@@ -0,0 +1,168 @@
+# Depends on:
+# - TEST_NAME
+# - TEST_S3_BUCKET
+# - TEST_S3_FOLDER
+
+# Workload configuration #######################################################
+#
+# Total CPU quota: 2290
+# Total MEM quota: 4580000
+# Total GPU quota: 40
+
+CLUSTER_URL="https://mw25dry.scaletesting.mesosphe.re/"
+SECURITY="strict"
+
+DCOS_CLI_URL="https://downloads.dcos.io/cli/releases/binaries/dcos/linux/x86-64/latest/dcos"
+
+# Test configuration ###########################################################
+
+SHOULD_INSTALL_INFRASTRUCTURE=true
+SHOULD_INSTALL_NON_GPU_DISPATCHERS=true
+SHOULD_INSTALL_GPU_DISPATCHERS=false
+SHOULD_RUN_FINITE_STREAMING_JOBS=true
+SHOULD_RUN_INFINITE_STREAMING_JOBS=true
+SHOULD_RUN_BATCH_JOBS=true
+SHOULD_RUN_GPU_BATCH_JOBS=false
+SHOULD_UNINSTALL_INFRASTRUCTURE_AT_THE_END=false
+
+# Infrastructure configuration #################################################
+
+GROUP_NAME="${TEST_NAME}"
+
+SERVICE_NAMES_PREFIX="${TEST_NAME}/"
+INFRASTRUCTURE_OUTPUT_FILE="infrastructure.json"
+
+KAFKA_CLUSTER_COUNT=1
+CASSANDRA_CLUSTER_COUNT=1
+
+ZOOKEEPER_CPUS=10
+ZOOKEEPER_MEM=20000
+ZOOKEEPER_CONFIG='scale-tests/configs/kafka-zookeeper-options.json'
+# Note: empty package repo values will default to latest Universe packages.
+ZOOKEEPER_PACKAGE_REPO=
+# 2.7.0-5.1.2e from the Universe.
+
+KAFKA_CPUS=10
+KAFKA_MEM=20000
+KAFKA_CONFIG='scale-tests/configs/kafka-options.json'
+# Note: empty package repo values will default to latest Universe packages.
+KAFKA_PACKAGE_REPO=
+# 2.9.0-5.4.0 from the Universe.
+
+CASSANDRA_CPUS=10
+CASSANDRA_MEM=20000
+CASSANDRA_CONFIG='scale-tests/configs/cassandra-options.json'
+# Note: empty package repo values will default to latest Universe packages.
+CASSANDRA_PACKAGE_REPO=
+# 2.9.0-3.11.6 from the Universe.
+
+# DSEngine configuration #######################################################
+
+DSENGINE_CPUS=10
+DSENGINE_MEM=20000
+DSENGINE_GPUS=40
+DSENGINE_PACKAGE_REPO=
+
+# Spark configuration ##########################################################
+
+SPARK_CONFIG='scale-tests/configs/spark-options.json'
+
+# Note: empty package repo values will default to latest Universe packages.
+# Spark version 2.10.0-2.4.5
+SPARK_PACKAGE_REPO=https://infinity-artifacts.s3.amazonaws.com/permanent/spark/2.10.0-2.4.5/stub-universe-spark.json
+
+# Note: leaving the Spark executor Docker image empty so that executors inherit
+# the image used for dispatchers.
+SPARK_EXECUTOR_DOCKER_IMAGE=
+
+# Non-GPU Spark dispatchers configuration ######################################
+
+# Not currently used.
+BATCH_MAX_NON_GPU_JOBS=1000
+
+SPARK_NON_GPU_DISPATCHERS=50
+SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE="non-gpu-dispatchers.out"
+# Note: this name is built internally by the deploy-dispatchers.py script.
+SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json"
+# Note: driver resources used per dispatcher (1 dispatcher will be able to run
+# 20 drivers since each driver requires 1 CPU).
+SPARK_NON_GPU_QUOTA_DRIVERS_CPUS=20
+SPARK_NON_GPU_QUOTA_DRIVERS_MEM=50000
+# Note: executor resources used per job (1 driver will run 1 job).
+SPARK_NON_GPU_QUOTA_EXECUTORS_CPUS=25
+SPARK_NON_GPU_QUOTA_EXECUTORS_MEM=40000
+
+# GPU Spark dispatchers configuration ##########################################
+
+# Not currently used.
+BATCH_MAX_GPU_JOBS=10
+
+SPARK_GPU_DISPATCHERS=0
+SPARK_GPU_DISPATCHERS_OUTPUT_FILE="gpu-dispatchers.out"
+SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json" # NOTE: this name is built internally by the deploy-dispatchers.py script.
+SPARK_GPU_QUOTA_DRIVERS_CPUS=
+SPARK_GPU_QUOTA_DRIVERS_MEM=
+SPARK_GPU_QUOTA_DRIVERS_GPUS=
+SPARK_GPU_QUOTA_EXECUTORS_CPUS=
+SPARK_GPU_QUOTA_EXECUTORS_MEM=
+SPARK_GPU_QUOTA_EXECUTORS_GPUS=
+SPARK_GPU_REMOVE_EXECUTORS_ROLES_QUOTAS=true
+# Common streaming jobs configuration ##########################################
+
+TEST_ASSEMBLY_JAR_URL='http://infinity-artifacts.s3.amazonaws.com/scale-tests/dcos-spark-scala-tests-assembly-2.4.0-20190325.jar'
+DISPATCHERS_JSON_OUTPUT_FILE="all-dispatchers.json"
+
+# Finite streaming jobs configuration ##########################################
+
+STREAMING_FINITE_SUBMISSIONS_OUTPUT_FILE="finite-submissions.out"
+STREAMING_FINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 50 dispatchers -> 50 producers.
+STREAMING_FINITE_CONSUMERS_PER_PRODUCER=1 # 50 producers -> 50 consumers.
+# 50 producers + 50 consumers = 100 total finite streaming jobs
+STREAMING_FINITE_PRODUCER_NUMBER_OF_WORDS=7692
+STREAMING_FINITE_PRODUCER_WORDS_PER_SECOND=1
+# 7692 words / 1 word per second -> ~2h runtime.
+STREAMING_FINITE_PRODUCER_SPARK_CORES_MAX=2
+STREAMING_FINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
+STREAMING_FINITE_CONSUMER_BATCH_SIZE_SECONDS=10
+STREAMING_FINITE_CONSUMER_SPARK_CORES_MAX=1
+STREAMING_FINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
+
+# Infinite streaming jobs configuration ########################################
+
+STREAMING_INFINITE_SUBMISSIONS_OUTPUT_FILE="infinite-submissions.out"
+STREAMING_INFINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 50 dispatchers -> 50 producers.
+STREAMING_INFINITE_CONSUMERS_PER_PRODUCER=1 # 50 producers -> 50 consumers.
+# 50 producers + 50 consumers = 100 total infinite streaming jobs
+STREAMING_INFINITE_PRODUCER_NUMBER_OF_WORDS=0
+STREAMING_INFINITE_PRODUCER_WORDS_PER_SECOND=1
+STREAMING_INFINITE_PRODUCER_SPARK_CORES_MAX=2
+STREAMING_INFINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
+STREAMING_INFINITE_CONSUMER_BATCH_SIZE_SECONDS=10
+STREAMING_INFINITE_CONSUMER_SPARK_CORES_MAX=1
+STREAMING_INFINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
+
+# Batch jobs configuration #####################################################
+
+SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
+
+BATCH_APP_ID="/${SERVICE_NAMES_PREFIX}batch-workload"
+BATCH_SCRIPT_CPUS=6
+BATCH_SCRIPT_MEM=12288
+BATCH_SUBMITS_PER_MIN=13
+# TODO: update to master for the next MWT.
+BATCH_SPARK_BUILD_BRANCH="dcos-58437-deploy-workloads-under-role-enforced-group"
+
+# Batch GPU jobs configuration #################################################
+
+SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
+
+GPU_APP_ID="/${SERVICE_NAMES_PREFIX}gpu-batch-workload"
+GPU_SCRIPT_CPUS=2
+GPU_SCRIPT_MEM=4096
+GPU_DOCKER_IMAGE='samvantran/spark-dcos-gpu:metrics'
+GPU_SUBMITS_PER_MIN=5
+GPU_MAX_DISPATCHERS=${SPARK_GPU_DISPATCHERS}
+GPU_SPARK_CORES_MAX=4
+GPU_SPARK_MESOS_EXECUTOR_GPUS=4
+GPU_SPARK_MESOS_MAX_GPUS=4
+GPU_SPARK_BUILD_BRANCH=master