From 6b116a61aa9bb33776b403db77113bf51c1655fe Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Thu, 21 Aug 2014 14:24:21 +0200 Subject: [PATCH 1/4] Add SSDs to block device mapping --- ec2/spark_ec2.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 3a8c816cfffa1..f2fdf6270ac6d 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -34,7 +34,7 @@ from optparse import OptionParser from sys import stderr import boto -from boto.ec2.blockdevicemapping import BlockDeviceMapping, EBSBlockDeviceType +from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType, EBSBlockDeviceType from boto import ec2 # A URL prefix from which to fetch AMI information @@ -342,6 +342,13 @@ def launch_cluster(conn, opts, cluster_name): device.delete_on_termination = True block_map["/dev/sdv"] = device + sdb = BlockDeviceType() + sdb.ephemeral_name = 'ephemeral0' + block_map['/dev/sdb'] = sdb + sdc = BlockDeviceType() + sdc.ephemeral_name = 'ephemeral1' + block_map['/dev/sdc'] = sdc + # Launch slaves if opts.spot_price is not None: # Launch spot instances with the requested price From e0d9e37d24ad71ec1448a750f263b66d2c583f6f Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Fri, 29 Aug 2014 11:31:13 +0200 Subject: [PATCH 2/4] Create ephemeral device mapping based on get_num_disks(). --- ec2/spark_ec2.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index f2fdf6270ac6d..d51ffcf07e3fe 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -26,6 +26,7 @@ import pipes import random import shutil +import string import subprocess import sys import tempfile @@ -342,12 +343,13 @@ def launch_cluster(conn, opts, cluster_name): device.delete_on_termination = True block_map["/dev/sdv"] = device - sdb = BlockDeviceType() - sdb.ephemeral_name = 'ephemeral0' - block_map['/dev/sdb'] = sdb - sdc = BlockDeviceType() - sdc.ephemeral_name = 'ephemeral1' - block_map['/dev/sdc'] = sdc + # Add ephemeral drives to device mapping. + for i in range(get_num_disks(opts.instance_type)): + dev = BlockDeviceType() + dev.ephemeral_name = 'ephemeral{}'.format(i) + # The first ephemeral drive is /dev/sdb. + name = '/dev/sd' + string.letters[i + 1] + block_map[name] = dev # Launch slaves if opts.spot_price is not None: From a1854d784ff85a4c8acfc0b004098ac2acb3b028 Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Fri, 29 Aug 2014 11:39:39 +0200 Subject: [PATCH 3/4] Only specify ephemeral device mapping for M3. --- ec2/spark_ec2.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index d51ffcf07e3fe..7c0dea942739e 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -343,13 +343,14 @@ def launch_cluster(conn, opts, cluster_name): device.delete_on_termination = True block_map["/dev/sdv"] = device - # Add ephemeral drives to device mapping. - for i in range(get_num_disks(opts.instance_type)): - dev = BlockDeviceType() - dev.ephemeral_name = 'ephemeral{}'.format(i) - # The first ephemeral drive is /dev/sdb. - name = '/dev/sd' + string.letters[i + 1] - block_map[name] = dev + # AWS ignores the AMI-specified block device mapping for M3. + if opts.instance_type.startswith('m3.'): + for i in range(get_num_disks(opts.instance_type)): + dev = BlockDeviceType() + dev.ephemeral_name = 'ephemeral{}'.format(i) + # The first ephemeral drive is /dev/sdb. + name = '/dev/sd' + string.letters[i + 1] + block_map[name] = dev # Launch slaves if opts.spot_price is not None: From 1ceb2c8a8fc3527cb3d7389f009ad2a1709f4ac1 Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Mon, 1 Sep 2014 12:28:17 +0200 Subject: [PATCH 4/4] Use %d string interpolation instead of {}. --- ec2/spark_ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 7c0dea942739e..a11a3e8c13328 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -347,7 +347,7 @@ def launch_cluster(conn, opts, cluster_name): if opts.instance_type.startswith('m3.'): for i in range(get_num_disks(opts.instance_type)): dev = BlockDeviceType() - dev.ephemeral_name = 'ephemeral{}'.format(i) + dev.ephemeral_name = 'ephemeral%d' % i # The first ephemeral drive is /dev/sdb. name = '/dev/sd' + string.letters[i + 1] block_map[name] = dev