From 508e0bc465fe9ca55729c1937667074e75fc16d4 Mon Sep 17 00:00:00 2001 From: maany Date: Fri, 21 Feb 2020 18:18:52 +0100 Subject: [PATCH] #177 retry leaving and joining swarm --- manifests/ccm_function/config_orchestrator.pp | 4 +- manifests/components/component_repository.pp | 14 +++---- manifests/components/swarm.pp | 38 +++++++++++++------ manifests/deploy/config_master/init.pp | 2 +- manifests/deploy/config_master/rollback.pp | 2 +- manifests/pre_deploy/config_master/init.pp | 6 +-- .../pre_deploy/config_master/rollback.pp | 4 +- .../lightweight_component/rollback.pp | 7 +++- 8 files changed, 50 insertions(+), 27 deletions(-) diff --git a/manifests/ccm_function/config_orchestrator.pp b/manifests/ccm_function/config_orchestrator.pp index 41d3c46..84ce1b9 100644 --- a/manifests/ccm_function/config_orchestrator.pp +++ b/manifests/ccm_function/config_orchestrator.pp @@ -51,13 +51,15 @@ class {'simple_grid::components::swarm::join': token => $manager_token, main_manager => $main_manager, + role => 'manager' } } elsif $fqdn == $main_manager{ notify{'Not executing docker swarm join command as the node is the main swarm manager':} }else { class {'simple_grid::components::swarm::join': token => $worker_token, - main_manager => $main_manager + main_manager => $main_manager, + role => 'worker' } } } diff --git a/manifests/components/component_repository.pp b/manifests/components/component_repository.pp index 4bf733d..827d886 100644 --- a/manifests/components/component_repository.pp +++ b/manifests/components/component_repository.pp @@ -118,7 +118,7 @@ user => root, logoutput => true, path => '/usr/sue/sbin:/usr/sue/bin:/use/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/sbin:/bin:/opt/puppetlabs/bin', - environment => ["HOME=/root"] + environment => ['HOME=/root'] } if $remove_images { $current_lightweight_component = simple_grid::get_lightweight_component($augmented_site_level_config_file, $execution_id) @@ -144,14 +144,14 @@ user => root, logoutput => true, path => '/usr/sue/sbin:/usr/sue/bin:/use/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/sbin:/bin:/opt/puppetlabs/bin', - environment => ["HOME=/root"] + environment => ['HOME=/root'] } exec{"Removing pre_config image: ${pre_config_image_name}": command => "docker rmi ${pre_config_image_name}", user => root, logoutput => true, path => '/usr/sue/sbin:/usr/sue/bin:/use/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/sbin:/bin:/opt/puppetlabs/bin', - environment => ["HOME=/root"] + environment => ['HOME=/root'] } } simple_grid::set_execution_status($deploy_status_file, $execution_id, $pending_deploy_status) @@ -182,7 +182,7 @@ path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', user => 'root', logoutput => true, - environment => ["HOME=/root"], + environment => ['HOME=/root'], timeout => 0, provider => shell } @@ -346,7 +346,7 @@ # path => "/usr/local/bin:/usr/bin/:/bin:/opt/puppetlabs/bin", # user => "root", # logoutput => true, - # environment => ["HOME=/root"] + # environment => ['HOME=/root'] # } } } @@ -372,7 +372,7 @@ # command => $command, # path => "/usr/local/bin:/usr/bin/:/bin:/opt/puppetlabs/bin", # user => "root", - # environment => ["HOME=/root"], + # environment => ['HOME=/root'], # provider => 'shell', # } } @@ -403,7 +403,7 @@ # command => $command, # path => "/usr/local/bin:/usr/bin/:/bin:/opt/puppetlabs/bin", # user => "root", - # environment => ["HOME=/root"] + # environment => ['HOME=/root'] # } } } diff --git a/manifests/components/swarm.pp b/manifests/components/swarm.pp index 7eecbf7..7499583 100644 --- a/manifests/components/swarm.pp +++ b/manifests/components/swarm.pp @@ -55,29 +55,32 @@ $bolt_cmd = "bolt task run docker::swarm_init --targets ${main_manager}" $bolt_token_cmd = "bolt task run simple_grid::swarm_prep_tokens main_manager=${main_manager} swarm_status_file=${swarm_status_file} --targets localhost" exec { 'Initialize Docker Swarm': - command => $bolt_cmd, - user => 'root', - logoutput => true, - path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', - environment => ["HOME=/root"] + command => $bolt_cmd, + user => 'root', + logoutput => true, + path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', + environment => ['HOME=/root'] } ~> exec { 'Save tokens for Docker Swarm': command => $bolt_token_cmd, user => 'root', logoutput => true, path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', - environment => ["HOME=/root"] + environment => ['HOME=/root'] } } # Executes on LC class simple_grid::components::swarm::join( - $swarm_status_file = lookup("simple_grid::components::swarm::status_file"), - $modulepath = lookup("simple_grid::components::ccm::install::modules_dir"), $token, - $main_manager + $main_manager, + $role, + $retry_wrapper = lookup('simple_grid::scripts::wrapper::retry'), + $wrapper_dir = lookup('simple_grid::scripts::wrapper_dir'), + ){ - $join_cmd = "docker swarm join --token=${token} ${main_manager}" + $join_cmd = "${wrapper_dir}/${retry_wrapper} --command='docker swarm join --token=${token} ${main_manager}' \ + --recovery-command='docker swarm leave --force' --reattempt-interval=10" exec { "Join swarm cluster as ${role}": command => $join_cmd, user => 'root', @@ -94,7 +97,7 @@ $ingress_gateway = lookup('simple_grid::components::swarm::ingress::gateway'), $ingress_network_name = lookup('simple_grid::components::swarm::ingress::name'), ){ - $ingress_rm_cmd = "yes | docker network rm ingress" + $ingress_rm_cmd = 'yes | docker network rm ingress' $ingress_create_cmd = "docker network create \ --driver overlay \ --ingress \ @@ -135,4 +138,17 @@ environment => ['HOME=/root'] } } + +class simple_grid::components::swarm::leave( + $retry_wrapper = lookup('simple_grid::scripts::wrapper::retry'), + $wrapper_dir = lookup('simple_grid::scripts::wrapper_dir'), +){ + $leave_command = "${wrapper_dir}/${retry_wrapper} --command='docker swarm leave --force' --reattempt-interval=5" + exec { 'Leave swarm cluster': + command => $leave_command, + user => 'root', + logoutput => true, + path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', + } +} ## TODO DNS info diff --git a/manifests/deploy/config_master/init.pp b/manifests/deploy/config_master/init.pp index 189327f..d1ffbdf 100644 --- a/manifests/deploy/config_master/init.pp +++ b/manifests/deploy/config_master/init.pp @@ -46,7 +46,7 @@ path => '/usr/local/bin/:/usr/bin/:/bin/:/opt/puppetlabs/bin/', user => 'root', logoutput => true, - environment => ["HOME=/root"], + environment => ['HOME=/root'], timeout => $timeout } } diff --git a/manifests/deploy/config_master/rollback.pp b/manifests/deploy/config_master/rollback.pp index 479bd7a..057b6dd 100644 --- a/manifests/deploy/config_master/rollback.pp +++ b/manifests/deploy/config_master/rollback.pp @@ -36,7 +36,7 @@ user => 'root', logoutput => true, timeout => $timeout, - environment => ["HOME=/root"] + environment => ['HOME=/root'] } simple_grid::components::execution_stage_manager::set_stage {"Setting stage to deploy": simple_stage => lookup('simple_grid::stage::deploy') diff --git a/manifests/pre_deploy/config_master/init.pp b/manifests/pre_deploy/config_master/init.pp index 1249d70..48718f3 100644 --- a/manifests/pre_deploy/config_master/init.pp +++ b/manifests/pre_deploy/config_master/init.pp @@ -39,7 +39,7 @@ path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', user => 'root', logoutput => true, - environment => ["HOME=/root"] + environment => ['HOME=/root'] } exec{"Running puppet agent on ${node_fqdn} to initiate step 2 of pre_deploy stage ": command => "bolt task run simple_grid::run_puppet_agent \ @@ -50,7 +50,7 @@ path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', user => 'root', logoutput => true, - environment => ["HOME=/root"] + environment => ['HOME=/root'] } exec{"Running puppet agent on ${node_fqdn} to initiate step 3 of pre_deploy stage ": @@ -62,7 +62,7 @@ path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', user => 'root', logoutput => true, - environment => ["HOME=/root"] + environment => ['HOME=/root'] } } elsif $mode == lookup('simple_grid::mode::release') { diff --git a/manifests/pre_deploy/config_master/rollback.pp b/manifests/pre_deploy/config_master/rollback.pp index 11e2d2a..9df07d8 100644 --- a/manifests/pre_deploy/config_master/rollback.pp +++ b/manifests/pre_deploy/config_master/rollback.pp @@ -27,7 +27,7 @@ path => '/usr/sue/sbin:/usr/sue/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/puppetlabs/bin', user => 'root', logoutput => true, - environment => ["HOME=/root"] + environment => ['HOME=/root'] } } elsif $mode == lookup('simple_grid::mode::release') { @@ -60,7 +60,7 @@ path => '/usr/local/bin/:/usr/bin/:/bin/:/opt/puppetlabs/bin/', user => 'root', logoutput => true, - environment => ["HOME=/root"] + environment => ['HOME=/root'] } } diff --git a/manifests/pre_deploy/lightweight_component/rollback.pp b/manifests/pre_deploy/lightweight_component/rollback.pp index 327cd3d..25249e7 100644 --- a/manifests/pre_deploy/lightweight_component/rollback.pp +++ b/manifests/pre_deploy/lightweight_component/rollback.pp @@ -8,6 +8,11 @@ $swarm_status_file = lookup('simple_grid::components::swarm::status_file'), $simple_log_dir = lookup('simple_grid::simple_log_dir'), ){ + # Use if you wish to deprecate the rollback_swarm task + # notify{'Leaving swarm':} + # class{'simple_grid::components::swarm::leave': + # before => File['Removing directory for scripts'] + # } file{"Removing augmented site level configuration file from LC": ensure => absent, @@ -27,7 +32,7 @@ path => "${host_certificates_dir}", } - file{"Removing directory for lifecycle callback scripts": + file{'Removing directory for scripts': ensure => absent, force => true, path => "${simple_config_dir}/${lifecycle_callbacks_dir_name}"