group updates to public branch

ansible.cfg

@@ -8,10 +8,14 @@
 force_valid_group_names = always
 force_color = 1
 
-#callbacks_enabled = yaml,profile_tasks,timer
-callbacks_enabled = yaml,timer
+#deprecation_warnings = False
 display_skipped_hosts = false
 
+#callbacks_enabled = yaml,profile_tasks,timer
+callbacks_enabled = yaml,profile_tasks
+#callbacks_enabled = yaml,timer
+#callbacks_enabled = yaml
+
 ## https://www.digitalocean.com/community/tutorials/how-to-manage-multistage-environments-with-ansible
 ## ref: https://github.com/ansible/ansible/issues/5316#issuecomment-30749291
 ## ref: https://serverfault.com/questions/628989/how-to-set-default-ansible-username-password-for-ssh-connection

files/scripts/bashenv/.bash_env

@@ -37,6 +37,9 @@ export EDITOR=/usr/local/bin/emacs
 export PYTHON_VERSION="3"
 export TEST_PYTHON_VERSION="3.10"
 
+## ref: https://pythonspeed.com/articles/docker-buildkit/
+export DOCKER_BUILDKIT=1
+
 if [[ "$platform" =~ ^(MSYS|MINGW64|MINGW32)$ ]]; then
     echo "${log_prefix} setting env for MSYS/MINGW platform"
 

inventory/group_vars/docker_stack.yml

@@ -176,17 +176,15 @@ docker_stack__compose_http_timeout: 240
 
 docker_source_images: []
 
-##########
-## stepca
-##########
-
 #docker_stack__stepca_image: "media.johnson.int:5000/docker-stepca:latest"
 
 ##########
 ## other
 ##########
 docker_stack__vmware_data_dir: /data/datacenter/vmware
 
+
+
 ##########
 ## registry
 ##########

inventory/group_vars/docker_stack_auth.yml

@@ -12,6 +12,8 @@ docker_stack__auth_google_client_secret: "{{ vault__docker_stack__auth_google_cl
 ##########
 ## authelia
 ##########
+docker_stack__auth_oauth_secret: "{{ vault__docker_stack__auth_oauth_secret }}"
+docker_stack__auth_whitelist: "{{ vault__docker_stack__auth_whitelist }}"
 
 docker_stack__authelia_jwt_secret: "{{ vault__docker_stack__authelia_jwt_secret | d('') }}"
 docker_stack__authelia_encryption_key: "{{ vault__docker_stack__authelia_encryption_key | d('') }}"
@@ -84,7 +86,7 @@ docker_stack__authelia_authentication_backend:
     implementation: custom
 
     # The url to the ldap server. Scheme can be ldap or ldaps in the format (port optional) <scheme>://<address>[:<port>].
-    url: "{{ ldap_uri }}"
+    url: "{{ ldap_internal_uri }}"
 
     # Use StartTLS with the LDAP connection.
     start_tls: false
@@ -100,7 +102,7 @@ docker_stack__authelia_authentication_backend:
       minimum_version: TLS1.2
 
     # The base dn for every entries.
-    base_dn: "{{ ldap_base }}"
+    base_dn: "{{ ldap_internal_base }}"
 
     # The attribute holding the username of the user. This attribute is used to populate
     # the username in the session information. It was introduced due to #561 to handle case

inventory/group_vars/docker_stack_control.yml

@@ -88,8 +88,8 @@ docker_stack__wordpress_backend: "http://wordpress.johnson.int:80"
 docker_stack__jenkins_ldap_host: "{{ ldap_internal_host }}"
 docker_stack__jenkins_ldap_base_dn: "{{ ldap_internal_base_dn }}"
 
-#docker_stack__jenkins_image: "media.johnson.int:5000/jenkins-docker:latest"
-docker_stack__jenkins_image: "media.johnson.int:5000/docker-jenkins:latest"
+docker_stack__jenkins_image: "{{ docker_stack__registry_endpoint }}/docker-jenkins:latest"
+#docker_stack__jenkins_image: "{{ docker_stack__registry_endpoint }}/docker-jenkins-jcac:latest"
 
 docker_stack__jenkins_agent_image: "media.johnson.int:5000/jenkins-docker-cicd-agent:latest"
 #docker_stack__jenkins_agent_jenkins_url: "https://jenkins.admin.johnson.int"

inventory/group_vars/docker_stack_jenkins_controller.yml

@@ -1,24 +1,42 @@
 ---
 
+###########
+## JENKINS
+###########
 docker_stack__service_groups__jenkins:
   - jenkins
 
+#docker_stack__jenkins_pipeline_lib_repo: "https://gitea.admin.dettonville.int/infra/pipeline-automation-lib.git"
+docker_stack__jenkins_pipeline_lib_repo: "ssh://git@gitea.admin.dettonville.int:2222/infra/pipeline-automation-lib.git"
+
+#docker_stack__jenkins_mgr_pwd_secret: "{{ vault__docker_stack__jenkins_mgr_pwd_secret | d('') }}"
+docker_stack__jenkins_mgr_pwd_secret: "{{ ldap_readonly_password }}"
+
+docker_stack__jenkins__agent_username: "{{ vault__jenkins_agent_username }}"
+docker_stack__jenkins__agent_password: "{{ vault__jenkins_agent_password }}"
+
 #docker_stack__proxy_services: []
 
 #docker_stack__volumes: []
 
 #external_domain: "{{ external_root_domain }}"
 #docker_stack__external_domain: "{{ external_domain }}"
 
-## JENKINS
-docker_stack__jenkins_ldap_host: "{{ docker_stack__ldap_host }}"
-docker_stack__jenkins_ldap_base_dn: "{{ docker_stack__ldap_base_dn }}"
+docker_registry_login: yes
+docker_stack__registry_enable_passthru: no
+#docker_stack__firewalld_enabled: no
 
-#docker_stack__jenkins_image: "{{ docker_stack__registry_endpoint }}/jenkins-docker:latest"
 #docker_stack__jenkins_image: "lj020326/docker-jenkins:latest"
-docker_stack__jenkins_image: "media.johnson.int:5000/docker-jenkins:latest"
+#docker_stack__jenkins_image: "media.johnson.int:5000/docker-jenkins:latest"
+docker_stack__jenkins_image: "{{ docker_stack__registry_endpoint }}/docker-jenkins:latest"
+#docker_stack__jenkins_image: "{{ docker_stack__registry_endpoint }}/docker-jenkins-jcac:latest"
+docker_stack__jenkins_agent_image: "{{ docker_stack__registry_endpoint }}/jenkins-docker-cicd-agent:latest"
+#docker_stack__jenkins_agent_image: "media.johnson.int:5000/jenkins-swarm-agent:latest"
 
-docker_stack__jenkins_mgr_pwd_secret: "{{ vault__docker_stack__jenkins_mgr_pwd_secret | d('') }}"
+docker_stack__jenkins_agent_jenkins_host: "jenkins.{{ docker_stack__internal_domain }}"
+
+docker_stack__jenkins_ldap_host: "{{ docker_stack__ldap_host }}"
+docker_stack__jenkins_ldap_base_dn: "{{ docker_stack__ldap_base_dn }}"
 
 ## moved to group_vars/docker_stack.yml
 #docker_stack__jenkins_agent_secret: "{{ vault__docker_stack__jenkins_agent_secret | d('') }}"
@@ -27,5 +45,26 @@ docker_stack__registry_service_common_name: "{{ docker_stack__internal_domain }}
 
 docker_stack__registry_service_data_path: "/data/docker_registry_local"
 
-docker_registry_login: yes
-docker_stack__registry_enable_passthru: no
+docker_stack__jenkins_vmware_data_dir: "{{ docker_stack__vmware_data_dir }}"
+
+docker_stack__jenkins_ssh_private_key: "{{ vault__git_ssh_private_keyfile }}"
+docker_stack__jenkins_ssh_public_key: "{{ vault__git_ssh_public_keyfile }}"
+
+docker_stack__jenkins_cred_jenkins_admin_user_password: "{{ vault__jenkins_cred_jenkins_admin_user_password }}"
+docker_stack__jenkins_cred_jenkins_git_user_password: "{{ vault__jenkins_cred_jenkins_git_user_password }}"
+docker_stack__jenkins_cred_ansible_vault_password: "{{ vault__ansible_vault_password }}"
+docker_stack__jenkins_cred_ansible_ssh_key: "{{ vault__jenkins_cred_ansible_ssh_key }}"
+
+docker_stack__jenkins_cred_ansible_ssh_username: "{{ vault__jenkins_cred_ansible_ssh_username }}"
+docker_stack__jenkins_cred_ansible_ssh_password: "{{ vault__jenkins_cred_ansible_ssh_password }}"
+
+docker_stack__jenkins_cred_vsphere_username: "{{ vault__jenkins_cred_vsphere_username }}"
+docker_stack__jenkins_cred_vsphere_password: "{{ vault__jenkins_cred_vsphere_password }}"
+docker_stack__jenkins_cred_esxi_password: "{{ vault__esxi_password }}"
+docker_stack__jenkins_cred_bitbucket_ssh_username: "{{ vault__jenkins_cred_bitbucket_ssh_username }}"
+docker_stack__jenkins_cred_bitbucket_ssh_private_key: "{{ vault__jenkins_cred_bitbucket_ssh_private_key }}"
+docker_stack__jenkins_cred_packer_ssh_password: "{{ vault__packer_user_password }}"
+docker_stack__jenkins_cred_vm_root_password: "{{ vault__sha512_hashed_root_password }}"
+docker_stack__jenkins_cred_github_username: "{{ vault__jenkins_cred_github_username }}"
+docker_stack__jenkins_cred_github_password: "{{ vault__jenkins_cred_github_password }}"
+docker_stack__jenkins_cred_docker_registry_admin_password: "{{ vault__jenkins_cred_docker_registry_admin_password }}"

inventory/group_vars/docker_stack_jenkins_jcac.yml

@@ -30,10 +30,8 @@ docker_registry_login: yes
 docker_stack__registry_enable_passthru: no
 #docker_stack__firewalld_enabled: no
 
-docker_stack__jenkins_jcac_image: "media.johnson.int:5000/docker-jenkins-jcac:latest"
-docker_stack__jenkins_jcac_agent_image: "media.johnson.int:5000/jenkins-docker-cicd-agent:latest"
-
-docker_stack__jenkins_mgr_pwd_secret: "{{ vault__docker_stack__jenkins_mgr_pwd_secret | d('') }}"
+docker_stack__jenkins_jcac_image: "{{ docker_stack__registry_endpoint }}/docker-jenkins-jcac:latest"
+docker_stack__jenkins_jcac_agent_image: "{{ docker_stack__registry_endpoint }}/jenkins-docker-cicd-agent:latest"
 
 docker_stack__jenkins_agent_jenkins_host: "jenkins.{{ docker_stack__internal_domain }}"
 

roles/bootstrap-webmin/tasks/install-debian.yml

@@ -18,8 +18,8 @@
 ## The "signed-by" repo option is available since Debian Stretch
 ## ref: https://github.com/ansible/ansible/issues/78063#issuecomment-1242970339
 - name: webmin | Debian | Set trust key type
-  when: (ansible_distribution=='Ubuntu' and ansible_distribution_major_version|int is version('18', '<=')) or
-    (ansible_distribution=='Debian' and ansible_distribution_major_version|int is version('9', '<='))
+  when: (ansible_distribution=='Ubuntu' and ansible_distribution_major_version|int<=18) or
+    (ansible_distribution=='Debian' and ansible_distribution_major_version|int<=9)
   set_fact:
     __webmin_apt_repo_key_type: apt_key
     __webmin_repo_template: webmin.debian.repo.jessie.j2

roles/bootstrap-webmin/tasks/main.yml

@@ -31,17 +31,25 @@
   changed_when: no
 
 - name: "webmin | Check perl"
+  environment:
+   PERL_MM_USE_DEFAULT: 1
   shell: perl -e 'use FileHandle; print $FileHandle::VERSION'
   changed_when: no
   become: true
 
 - name: "webmin | Install webmin"
+  environment:
+   PERL_MM_USE_DEFAULT: 1
   include_tasks: "install-{{ ansible_os_family|lower }}.yml"
 
 - name: "webmin | Configure webmin users"
+  environment:
+   PERL_MM_USE_DEFAULT: 1
   when: bootstrap_webmin__users|d([])|length > 0
   include_tasks: setup-users.yml
 
 - name: "webmin | Download and install webmin modules"
+  environment:
+   PERL_MM_USE_DEFAULT: 1
   when: bootstrap_webmin__modules|d([])|length > 0
   include_tasks: setup-modules.yml

roles/bootstrap-webmin/templates/webmin.centos.repo.j2

@@ -1,7 +1,8 @@
 [Webmin]
 name=Webmin Distribution Neutral
 #baseurl=http://download.webmin.com/download/yum
-mirrorlist=http://download.webmin.com/download/yum/mirrorlist
+mirrorlist=https://download.webmin.com/download/yum/mirrorlist
 enabled=1
-gpgcheck=1
+#gpgcheck=1
+gpgcheck=0
 gpgkey=http://www.webmin.com/jcameron-key.asc

roles/bootstrap-webmin/vars/centos-8.yml

@@ -2,6 +2,9 @@
 
 bootstrap_webmin__base_dir: "/usr/libexec/webmin"
 
+bootstrap_webmin__packages:
+  - perl
+
 #bootstrap_webmin__packages:
 #  - perl
 #  - perl-Env

roles/docker-stack/tasks/init-vars.yml

@@ -171,25 +171,28 @@
        var: __docker_stack__proxy_services
        verbosity: 2
 
-  - name: "{{ log_prefix_local }} Set the app proxy services vars"
-    include_vars:  "app-services/common/docker_stack__proxy_{{ item.name }}.yml"
+  - name: "{{ log_prefix_local }} Set the app proxy services vars specific to traefik version"
+    when: (role_path + '/vars/app-services/traefik/' + docker_stack__traefik_version + '/' + docker_stack__traefik_routing_method + '/docker_stack_proxy_' + item.name + '.yml') is file
+    include_vars:  "app-services/traefik/{{ docker_stack__traefik_version }}/{{ docker_stack__traefik_routing_method }}/docker_stack_proxy_{{ item.name }}.yml"
     with_items: "{{ __docker_stack__proxy_services }}"
 
-  - name: "{{ log_prefix_local }} Set the app proxy services vars specific to traefik version"
-    include_vars:  "app-services/traefik/{{ docker_stack__traefik_version }}/{{ docker_stack__traefik_routing_method }}/docker_stack__proxy_{{ item.name }}.yml"
+  - name: "{{ log_prefix_local }} Set the app proxy services vars"
+    when: (role_path + '/vars/app-services/common/docker_stack_proxy_' + item.name + '.yml') is file
+    include_vars:  "app-services/common/docker_stack_proxy_{{ item.name }}.yml"
     with_items: "{{ __docker_stack__proxy_services }}"
 
-- name: "{{ log_prefix_local }} Set the common app services vars"
+- name: "{{ log_prefix_local }} Set the traefik app services vars"
 #  when: item.source|d('role') == 'role'
-  when: (role_path + '/vars/app-services/common/docker_stack_' + item.name + '.yml') is file
-  include_vars: "app-services/common/docker_stack_{{ item.name }}.yml"
+  when: (role_path + '/vars/app-services/traefik/' + docker_stack__traefik_version + '/' + docker_stack__traefik_routing_method + '/docker_stack_' + item.name + '.yml') is file
+  include_vars:  "app-services/traefik/{{ docker_stack__traefik_version }}/{{ docker_stack__traefik_routing_method }}/docker_stack_{{ item.name }}.yml"
   loop: "{{ __docker_stack__service_groups }}"
   loop_control:
     label: "{{ item.name }}"
 
-- name: "{{ log_prefix_local }} Set the app services vars specific to traefik version"
-  when: item.source|d('role') == 'role'
-  include_vars:  "app-services/traefik/{{ docker_stack__traefik_version }}/{{ docker_stack__traefik_routing_method }}/docker_stack_{{ item.name }}.yml"
+- name: "{{ log_prefix_local }} Set the common app services vars"
+#  when: item.source|d('role') == 'role'
+  when: (role_path + '/vars/app-services/common/docker_stack_' + item.name + '.yml') is file
+  include_vars: "app-services/common/docker_stack_{{ item.name }}.yml"
   loop: "{{ __docker_stack__service_groups }}"
   loop_control:
     label: "{{ item.name }}"
@@ -211,13 +214,13 @@
 - name: "{{ log_prefix_local }} Combine app varname configs into __docker_stack__app_configs"
   set_fact:
     __docker_stack__app_configs: "{{ __docker_stack__app_configs|d({})
-      | combine({ item.name: lookup('vars', q('varnames','^docker_stack__appspec__'+item.name.replace('-','_')+'$')| first | d('') )|d({}) }) }}"
+      | combine({ item.name: lookup('vars', q('varnames','^__docker_stack__appspec__'+item.name.replace('-','_')+'$')| first | d('') )|d({}) }) }}"
   loop: "{{ __docker_stack__service_groups }}"
 
 ## NOTE: sensitive info - this var contains the secrets
-#- name: "{{ log_prefix_local }} Display __docker_stack__app_configs"
-#  debug:
-#     var: __docker_stack__app_configs
+- name: "{{ log_prefix_local }} Display __docker_stack__app_configs"
+  debug:
+     var: __docker_stack__app_configs
 ##     verbosity: 3
 
 - name: "{{ log_prefix_local }} Display __docker_stack__proxy_services"
@@ -230,12 +233,12 @@
 
 - name: "{{ log_prefix_local }} Show variables with 'docker_stack__proxyspec__' prefix"
   debug:
-    msg: "{{ lookup('varnames', '^docker_stack__proxyspec__', wantlist=True) }}"
+    msg: "{{ lookup('varnames', '^__docker_stack__proxyspec__', wantlist=True) }}"
 
 - name: "{{ log_prefix_local }} Combine proxy varname configs into __docker_stack__proxy_configs"
   set_fact:
     __docker_stack__proxy_configs: "{{ __docker_stack__proxy_configs|d({})
-      | combine({ item.name: lookup('vars', q('varnames','^docker_stack__proxyspec__'+item.name.replace('-','_')+'$')| first | d('') )|d({}) }) }}"
+      | combine({ item.name: lookup('vars', q('varnames','^__docker_stack__proxyspec__'+item.name.replace('-','_')+'$')| first | d('') )|d({}) }) }}"
   loop: "{{ __docker_stack__proxy_services|d([]) }}"
 
 - name: "{{ log_prefix_local }} Display __docker_stack__proxy_configs"
@@ -253,7 +256,7 @@
 - name: "{{ log_prefix_local }} Display __docker_stack__proxy_service_configs"
   debug:
      var: __docker_stack__proxy_service_configs
-     verbosity: 2
+#     verbosity: 2
 
 - name: "{{ log_prefix_local }} Init __docker_stack__secrets"
   set_fact:

roles/docker-stack/tasks/main.yml

@@ -70,6 +70,10 @@
       include_tasks: setup-service-configs.yml
 
   rescue:
+    - name: "{{ log_prefix_local }} Init __restart_docker_service"
+      set_fact:
+        __restart_docker_service: no
+
     - name: "{{ log_prefix_local }} Display docker stack config results"
       when:
         - item is defined

roles/docker-stack/tasks/run-compose-action.yml

@@ -47,12 +47,13 @@
 
 - name: "{{ log_prefix_local }} Display compose args"
   debug:
-    msg:
-      - "__docker_stack__compose_state: {{ __docker_stack__compose_state }}"
-      - "__docker_stack__compose_restarted: {{ __docker_stack__compose_restarted|d('') }}"
-      - "__docker_stack__compose_stopped: {{ __docker_stack__compose_stopped|d('') }}"
-      - "__docker_stack__service_groups: {{ __docker_stack__service_groups|d('') }}"
-      - "__docker_stack__app_services: {{ __docker_stack__app_services }}"
+    var: "{{ item }}"
+  loop:
+      - __docker_stack__compose_state
+      - __docker_stack__compose_restarted
+      - __docker_stack__compose_stopped
+      - __docker_stack__service_groups
+      - __docker_stack__app_services
 
 - name: "{{ log_prefix_local }} Start the docker stack"
   block:
@@ -92,6 +93,10 @@
         var: __compose_result
         verbosity: 1
 
+    - name: "{{ log_prefix_local }} Init __restart_docker"
+      set_fact:
+        __restart_docker: no
+
     - name: "{{ log_prefix_local }} Set __restart_docker"
       set_fact:
 #        __restart_docker: "{{ (__compose_result.msg | regex_search("INVALID_ZONE .* docker")) }}"

roles/docker-stack/tasks/setup-app-configs.yml

@@ -35,6 +35,11 @@
     recurse: "{{ item.recurse | d( omit ) }}"
   with_items: "{{ __docker_stack__app_config_dirs }}"
 
+- name: "{{ log_prefix_local }} Display __docker_stack__proxy_service_configs"
+  when: __docker_stack__proxy_services | length > 0
+  debug:
+    var: __docker_stack__proxy_service_configs
+
 - name: "{{ log_prefix_local }} Install docker stack proxy service templates"
   when: __docker_stack__proxy_services | length > 0
   template: