deal with ebs mount edge case; bug 1173085

puppet/modules/socorro/manifests/role/processor.pp

@@ -1,28 +1,55 @@
 # Set up a processor node.
 class socorro::role::processor {
 
-include socorro::role::common
+  include socorro::role::common
 
+  # Symbols uses a hierarchy of directories that should be on the same
+  # block device for performance reasons.
+  $symbols_base = '/mnt/symbols'
+  $symbols_dirs = [
+    "${symbols_base}/cache",
+    "${symbols_base}/tmp"
+  ]
+
+  file {
+    $symbols_base:
+      ensure => directory
+  }
+
+  # There is an edge-case interaction between cloud-init and *some* instance
+  # types that causes EBS volumes to be pre-mounted.
+  # https://bugzilla.mozilla.org/show_bug.cgi?id=1173085
   exec {
-    'format-symbol-cache':
+    'check-premounted-ebs':
+      path    => '/bin',
+      command => 'umount /dev/xvdb',
+      onlyif  => 'mount | grep xvdb'
+  }
+
+  exec {
+    'format-symbols-cache':
       path    => '/usr/sbin',
-      command => 'mkfs.ext4 /dev/xvdc'
+      command => 'mkfs.ext4 /dev/xvdb',
+      require => Exec['check-premounted-ebs']
   }
 
   mount {
-    '/mnt':
+    $symbols_base:
       ensure  => mounted,
-      device  => '/dev/xvdc',
+      device  => '/dev/xvdb',
       fstype  => 'ext4',
       options => 'defaults',
-      require => Exec['format-symbol-cache']
+      require => [
+        Exec['format-symbols-cache'],
+        File[$symbols_base]
+      ]
   }
 
   file {
-    '/mnt/symbolcache':
+    $symbols_dirs:
       ensure  => directory,
       owner   => 'socorro',
-      require => Mount['/mnt']
+      require => Mount[$symbols_base];
   }
 
   service {
@@ -31,7 +58,7 @@
       enable  => true,
       require => [
         Exec['join_consul_cluster'],
-        File['/mnt/symbolcache']
+        File[$symbols_dirs]
       ]
   }
 

terraform/processor/main.tf

@@ -71,9 +71,8 @@ resource "aws_launch_configuration" "lc-processor" {
     lifecycle {
         create_before_destroy = true
     }
-
     ephemeral_block_device {
-        device_name = "/dev/xvdc"
+        device_name = "/dev/xvdb"
         virtual_name = "ephemeral0"
     }
 }
@@ -94,18 +93,18 @@ resource "aws_autoscaling_group" "as-processor" {
     min_size = "${lookup(var.processor_num, var.environment)}"
     desired_capacity = "${lookup(var.processor_num, var.environment)}"
     tag {
-      key = "Environment"
-      value = "${var.environment}"
-      propagate_at_launch = true
+        key = "Environment"
+        value = "${var.environment}"
+        propagate_at_launch = true
     }
     tag {
-      key = "role"
-      value = "processor"
-      propagate_at_launch = true
+        key = "role"
+        value = "processor"
+        propagate_at_launch = true
     }
     tag {
-      key = "project"
-      value = "socorro"
-      propagate_at_launch = true
+        key = "project"
+        value = "socorro"
+        propagate_at_launch = true
     }
 }