Merge pull request #189 from simple-framework/issue_177

#177 retry leaving and joining swarm
simple-framework · Feb 21, 2020 · 199ab20 · 199ab20
2 parents 5e037b5 + 508e0bc
commit 199ab20
Show file tree

Hide file tree

Showing 8 changed files with 50 additions and 27 deletions.
diff --git a/manifests/ccm_function/config_orchestrator.pp b/manifests/ccm_function/config_orchestrator.pp
@@ -51,13 +51,15 @@
         class {'simple_grid::components::swarm::join':
           token        => $manager_token,
           main_manager => $main_manager,
+          role         => 'manager'
         }
       } elsif $fqdn == $main_manager{
           notify{'Not executing docker swarm join command as the node is the main swarm manager':}
       }else {
         class {'simple_grid::components::swarm::join':
           token        => $worker_token,
-          main_manager => $main_manager
+          main_manager => $main_manager,
+          role         => 'worker'
         }
       }
     }

diff --git a/manifests/components/component_repository.pp b/manifests/components/component_repository.pp
@@ -118,7 +118,7 @@
     user        => root,
     logoutput   => true,
     path        => '/usr/sue/sbin:/usr/sue/bin:/use/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/sbin:/bin:/opt/puppetlabs/bin',
-    environment => ["HOME=/root"]
+    environment => ['HOME=/root']
   }
   if $remove_images {
     $current_lightweight_component = simple_grid::get_lightweight_component($augmented_site_level_config_file, $execution_id)
@@ -144,14 +144,14 @@
       user        => root,
       logoutput   => true,
       path        => '/usr/sue/sbin:/usr/sue/bin:/use/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/sbin:/bin:/opt/puppetlabs/bin',
-      environment => ["HOME=/root"]
+      environment => ['HOME=/root']
     }
     exec{"Removing pre_config image: ${pre_config_image_name}":
       command     => "docker rmi ${pre_config_image_name}",
       user        => root,
       logoutput   => true,
       path        => '/usr/sue/sbin:/usr/sue/bin:/use/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/sbin:/bin:/opt/puppetlabs/bin',
-      environment => ["HOME=/root"]
+      environment => ['HOME=/root']
     }
   }
   simple_grid::set_execution_status($deploy_status_file, $execution_id, $pending_deploy_status)
@@ -182,7 +182,7 @@
         path        => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
         user        => 'root',
         logoutput   => true,
-        environment => ["HOME=/root"],
+        environment => ['HOME=/root'],
         timeout     => 0,
         provider    => shell
       }
@@ -346,7 +346,7 @@
     #   path    => "/usr/local/bin:/usr/bin/:/bin:/opt/puppetlabs/bin",
     #   user    => "root",
     #   logoutput => true,
-    #   environment => ["HOME=/root"]
+    #   environment => ['HOME=/root']
     # }
   }
 }
@@ -372,7 +372,7 @@
   #     command => $command,
   #     path    => "/usr/local/bin:/usr/bin/:/bin:/opt/puppetlabs/bin",
   #     user    => "root",
-  #     environment => ["HOME=/root"],
+  #     environment => ['HOME=/root'],
   #     provider => 'shell',
   # }
 }
@@ -403,7 +403,7 @@
     #   command => $command,
     #   path    => "/usr/local/bin:/usr/bin/:/bin:/opt/puppetlabs/bin",
     #   user    => "root",
-    #   environment => ["HOME=/root"]
+    #   environment => ['HOME=/root']
     # }
   }
 }
diff --git a/manifests/components/swarm.pp b/manifests/components/swarm.pp
@@ -55,29 +55,32 @@
   $bolt_cmd = "bolt task run docker::swarm_init --targets ${main_manager}"
   $bolt_token_cmd = "bolt task run simple_grid::swarm_prep_tokens main_manager=${main_manager} swarm_status_file=${swarm_status_file} --targets localhost"
   exec { 'Initialize Docker Swarm':
-    command   => $bolt_cmd,
-    user      => 'root',
-    logoutput => true,
-    path      => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
-    environment       => ["HOME=/root"]
+    command     => $bolt_cmd,
+    user        => 'root',
+    logoutput   => true,
+    path        => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
+    environment => ['HOME=/root']
   } ~>
   exec { 'Save tokens for Docker Swarm':
     command           => $bolt_token_cmd,
     user              => 'root',
     logoutput         => true,
     path              => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
-    environment       => ["HOME=/root"]
+    environment       => ['HOME=/root']
   }
 }
 
 # Executes on LC
 class simple_grid::components::swarm::join(
-  $swarm_status_file = lookup("simple_grid::components::swarm::status_file"),
-  $modulepath = lookup("simple_grid::components::ccm::install::modules_dir"),
   $token,
-  $main_manager
+  $main_manager,
+  $role,
+  $retry_wrapper = lookup('simple_grid::scripts::wrapper::retry'),
+  $wrapper_dir = lookup('simple_grid::scripts::wrapper_dir'),
+
 ){
-  $join_cmd = "docker swarm join --token=${token}  ${main_manager}"
+  $join_cmd = "${wrapper_dir}/${retry_wrapper} --command='docker swarm join --token=${token}  ${main_manager}' \
+  --recovery-command='docker swarm leave --force' --reattempt-interval=10"
   exec { "Join swarm cluster as ${role}":
     command   => $join_cmd,
     user      => 'root',
@@ -94,7 +97,7 @@
   $ingress_gateway = lookup('simple_grid::components::swarm::ingress::gateway'),
   $ingress_network_name = lookup('simple_grid::components::swarm::ingress::name'),
 ){
-  $ingress_rm_cmd = "yes | docker network rm ingress"
+  $ingress_rm_cmd = 'yes | docker network rm ingress'
   $ingress_create_cmd = "docker network create \
                                       --driver overlay \
                                       --ingress \
@@ -135,4 +138,17 @@
     environment => ['HOME=/root']
   }
 }
+
+class simple_grid::components::swarm::leave(
+  $retry_wrapper = lookup('simple_grid::scripts::wrapper::retry'),
+  $wrapper_dir = lookup('simple_grid::scripts::wrapper_dir'),
+){
+  $leave_command = "${wrapper_dir}/${retry_wrapper} --command='docker swarm leave --force' --reattempt-interval=5"
+  exec { 'Leave swarm cluster':
+    command   => $leave_command,
+    user      => 'root',
+    logoutput => true,
+    path      => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
+  }
+}
 ## TODO DNS info
diff --git a/manifests/deploy/config_master/init.pp b/manifests/deploy/config_master/init.pp
@@ -46,7 +46,7 @@
       path    => '/usr/local/bin/:/usr/bin/:/bin/:/opt/puppetlabs/bin/',
       user    => 'root',
       logoutput => true,
-      environment => ["HOME=/root"],
+      environment => ['HOME=/root'],
       timeout => $timeout
       }
  }
diff --git a/manifests/deploy/config_master/rollback.pp b/manifests/deploy/config_master/rollback.pp
@@ -36,7 +36,7 @@
       user      => 'root',
       logoutput => true,
       timeout   => $timeout,
-      environment => ["HOME=/root"]
+      environment => ['HOME=/root']
     }
     simple_grid::components::execution_stage_manager::set_stage {"Setting stage to deploy":
     simple_stage => lookup('simple_grid::stage::deploy')

diff --git a/manifests/pre_deploy/config_master/init.pp b/manifests/pre_deploy/config_master/init.pp
@@ -39,7 +39,7 @@
         path    => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
         user    => 'root',
         logoutput => true,
-        environment => ["HOME=/root"]
+        environment => ['HOME=/root']
       }
       exec{"Running puppet agent on ${node_fqdn} to initiate step 2 of pre_deploy stage ":
         command => "bolt task run simple_grid::run_puppet_agent \
@@ -50,7 +50,7 @@
         path    => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
         user    => 'root',
         logoutput => true,
-        environment => ["HOME=/root"]
+        environment => ['HOME=/root']
       }
 
       exec{"Running puppet agent on ${node_fqdn} to initiate step 3 of pre_deploy stage ":
@@ -62,7 +62,7 @@
         path    => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
         user    => 'root',
         logoutput => true,
-        environment => ["HOME=/root"]
+        environment => ['HOME=/root']
       }
     }
     elsif $mode == lookup('simple_grid::mode::release') {

diff --git a/manifests/pre_deploy/config_master/rollback.pp b/manifests/pre_deploy/config_master/rollback.pp
@@ -27,7 +27,7 @@
       path    => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin',
       user    => 'root',
       logoutput => true,
-      environment => ["HOME=/root"]
+      environment => ['HOME=/root']
     }
   }
   elsif $mode == lookup('simple_grid::mode::release') {
@@ -60,7 +60,7 @@
       path    => '/usr/local/bin/:/usr/bin/:/bin/:/opt/puppetlabs/bin/',
       user    => 'root',
       logoutput => true,
-      environment => ["HOME=/root"]
+      environment => ['HOME=/root']
     }
   }
 

diff --git a/manifests/pre_deploy/lightweight_component/rollback.pp b/manifests/pre_deploy/lightweight_component/rollback.pp
@@ -8,6 +8,11 @@
   $swarm_status_file = lookup('simple_grid::components::swarm::status_file'),
   $simple_log_dir = lookup('simple_grid::simple_log_dir'),
 ){
+  # Use if you wish to deprecate the rollback_swarm task
+  # notify{'Leaving swarm':}
+  # class{'simple_grid::components::swarm::leave':
+  #   before => File['Removing directory for scripts']
+  # }
 
   file{"Removing augmented site level configuration file from LC":
     ensure => absent,
@@ -27,7 +32,7 @@
       path => "${host_certificates_dir}",
   }
 
-  file{"Removing directory for lifecycle callback scripts":
+  file{'Removing directory for scripts':
    ensure => absent,
    force  => true,
    path => "${simple_config_dir}/${lifecycle_callbacks_dir_name}"