Implements #205 Installer must have public key initial auth capability (

#270) * ECS-CommunityEdition-205 Installer must have public key initial auth capability (cherry picked from commit 6eea10b5db3985f960d7b313d2e705a0f913ba55) * More sausage for the initial ssh key auth (cherry picked from commit 8535ccb5430e89b79d253ea1e74390a39b8b20f3) * more sausage (cherry picked from commit edf961e0765cd9a06ccea1a6d1e2406816533f46) * deploy.yml change ideas (cherry picked from commit ef48e2cc57fa6d0a57aa30bc62ba816fc167aed9) * bootstrap.sh modifications (cherry picked from commit d0b3c630f0a2004fe23534aca7e4a95986dce383) * bootstrap.sh modifications (cherry picked from commit 86f897af9395a57af5b33c2162cd85422b4e6ded) * move generic help to generic_help.j2.yml file from config.yml * include shipit.lib.sh * build install paths early add copy action for ssh PKI material * fix a couple gitopt bugs * add create_install_tree() to plugin-defaults.sh * copy ssh keys in bootstrap.sh * more longopts adjustments * add loop delay in retry_with_timeout() * stop trying to autoremove curl, it'll always error. * key_vals need basename not full path set 0700 bits on ssh/ssl stores * remove optarg debugging * more ssh pubkey sausage * update reference.deploy.yml to include feature * jenkins changes * jenkins changes * jenkins changes * jenkins changes * jenkins changes
EMCECS · Aug 22, 2017 · f7ce40c · f7ce40c
1 parent 57a1437
commit f7ce40c
Show file tree

Hide file tree

Showing 40 changed files with 506 additions and 234 deletions.
diff --git a/bootstrap.sh b/bootstrap.sh
diff --git a/bootstrap_plugins/centos72.plugin.sh b/bootstrap_plugins/centos72.plugin.sh
@@ -17,7 +17,8 @@ os_supported=true
 docker_binary='/bin/docker'
 
 # packages to clean up during preflight
-list_preflight_packages="git nfs-client rsync wget curl epel-release ntp docker vim rsync pigz gdisk aria2 htop iotop iftop multitail dstat jq python-docker-py dkms qemu-guest-agent open-vm-tools docker"
+# Don't `yum autoremove curl`.  Yum is a dependency and it will throw errors.
+list_preflight_packages="git nfs-client nfs-tools rsync wget ntp docker vim pigz gdisk aria2 htop iotop iftop multitail dstat jq python-docker-py dkms qemu-guest-agent open-vm-tools docker"
 #nfs-tools"
 
 # Do any OS-specific tasks that must be done prior to bootstrap
@@ -76,7 +77,7 @@ in_repo_pkg() {
 }
 
 rm_repo_pkg() {
-    retry_with_timeout 10 300 yum -y autoremove $*
+    retry_with_timeout 10 300 sudo yum -y autoremove $*
 }
 
 # command to update all packages in the os package manager
@@ -86,10 +87,9 @@ up_repo_pkg_all() {
 
 # command to rebuild the os package manager's database
 up_repo_db() {
-    retry_with_timeout 10 300 yum -y makecache
+    retry_with_timeout 10 300 sudo yum -y makecache
 }
 
-
 # command to set os package manager proxy
 set_repo_proxy_conf() {
     sudo sed -i -e '/^proxy=/d' /etc/yum.conf

diff --git a/bootstrap_plugins/centos73.plugin.sh b/bootstrap_plugins/centos73.plugin.sh
@@ -17,7 +17,8 @@ os_supported=true
 docker_binary='/bin/docker'
 
 # packages to clean up during preflight
-list_preflight_packages="git nfs-client rsync wget curl epel-release ntp docker vim rsync pigz gdisk aria2 htop iotop iftop multitail dstat jq python-docker-py dkms qemu-guest-agent open-vm-tools docker"
+# Don't `yum autoremove curl`.  Yum is a dependency and it will throw errors.
+list_preflight_packages="git nfs-client nfs-tools rsync wget ntp docker vim pigz gdisk aria2 htop iotop iftop multitail dstat jq python-docker-py dkms qemu-guest-agent open-vm-tools docker"
 
 # Do any OS-specific tasks that must be done prior to bootstrap
 do_preflight() {
@@ -75,7 +76,7 @@ in_repo_pkg() {
 }
 
 rm_repo_pkg() {
-    retry_with_timeout 10 300 yum -y autoremove $*
+    retry_with_timeout 10 300 sudo yum -y autoremove $*
 }
 
 # command to update all packages in the os package manager
@@ -85,7 +86,7 @@ up_repo_pkg_all() {
 
 # command to rebuild the os package manager's database
 up_repo_db() {
-    retry_with_timeout 10 300 yum -y makecache
+    retry_with_timeout 10 300 sudo yum -y makecache
 }
 
 # command to set os package manager proxy

diff --git a/bootstrap_plugins/plugin-defaults.plugin.sh b/bootstrap_plugins/plugin-defaults.plugin.sh
@@ -16,8 +16,11 @@ os_supported=false
 # Caller symlinks
 unset symlink_scripts
 symlink_scripts() {
-    symlinks="ecsdeploy catfacts enter pingnodes step1 step2 island-step1 island-step2 update_deploy update_image rebuild_image inventory testbook videploy"
-    symlinks="${symlinks} island-step3 ecsconfig ecsremove"
+    symlinks="ecsdeploy ecsconfig ecsremove catfacts enter pingnodes  update_deploy"
+    symlinks="${symlinks} update_image rebuild_image inventory testbook videploy"
+    symlinks="${symlinks} step1 step2"
+    symlinks="${symlinks} island-step1 island-step2 island-step3"
+    symlinks="${symlinks} ova-step1 ova-step2"
     mkdir -p "${HOME}/bin"
     for l in $symlinks; do
         ln -s "${root}/ui/run.sh" "${HOME}/bin/$l" 2>/dev/null
@@ -30,3 +33,9 @@ update_path_in_bashrc() {
     sudo sed -i -e '/PATH/d' $HOME/.bashrc
     echo 'export PATH=$PATH:$HOME/.local/bin:$HOME/bin' >> $HOME/.bashrc
 }
+
+create_install_tree() {
+    for pathname in "${ui_host_ssl_dir}" "${ui_host_ssh_dir}"; do
+        sudo mkdir -p "${pathname}"
+    done
+}
diff --git a/changelog.md b/changelog.md
@@ -1,3 +1,7 @@
+# See https://github.com/EMCECS/ECS-CommunityEdition/releases for changelog of current releases.
+
+These entries are included for historical record.
+
 ## Update 2017-04-30: v 3.0.0.1 (HF1)
 - Updated Docker image to [ECS Software v3.0 Hotfix 1]()
 - Go-live with Installer 2.0 and yaml-configured, Ansible-based deployment system

diff --git a/docs/design/reference.deploy.yml b/docs/design/reference.deploy.yml
@@ -1,4 +1,4 @@
-# deploy.yml reference implementation v2.1.0
+# deploy.yml reference implementation v2.2.0
 
 # [Optional]
 # By changing the license_accepted boolean value to "true" you are
@@ -33,10 +33,22 @@ facts:
   # gain initial access to each node in the deployment and set up ssh public key authentication.
   # If these are not correct, the deployment will fail.
   ssh_defaults:
-    # Username to login as
+    # [Required]
+    # Username to use when logging in to nodes
     ssh_username: admin
+    # [Required]
     # Password to use with SSH login
+    # *** Set to same value as ssh_username to enable SSH public key authentication ***
     ssh_password: ChangeMe
+    # [Required when enabling SSH public key authentication]
+    # Password to give to sudo when gaining root access.
+    ansible_become_pass: ChangeMe
+    # [Required]
+    # Select the type of crypto to use when dealing with ssh public key
+    # authentication. Valid values here are:
+    #  - "rsa" (Default)
+    #  - "ed25519"
+    ssh_crypto: rsa
 
   # [Required]
   # Environment configuration for this deployment.
@@ -57,14 +69,14 @@ facts:
     # [Optional]
     # Picklist for node names.
     # Available options:
-    # - "moons" (ECS CE default)
-    # - "cities" (ECS SKU-flavored)
+    #  - "moons" (ECS CE default)
+    #  - "cities" (ECS SKU-flavored)
     autonaming: moons
     #
     # [Optional]
     # If your ECS comes with differing default credentials, you can specify those here
-    ecs_root_user: root
-    ecs_root_pass: ChangeMe
+    # ecs_root_user: root
+    # ecs_root_pass: ChangeMe
 
   # [Optional]
   # Storage pool defaults. Configure to your liking.

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -12,6 +12,8 @@ Welcome to ECS Community Edition's documentation!
    :maxdepth: 1
 
    installation/ECS-Installation.rst
+   installation/deploy.yml.rst
+   installation/public-key-authentication.rst
    use/ECS-UI-Web-Interface.rst
    use/Migration.rst
    troubleshooting/ECS-Troubleshooting.rst

diff --git a/docs/source/installation/public-key-authentication.md b/docs/source/installation/public-key-authentication.md
diff --git a/tests/ansible/roles/install_node/tasks/main.yml b/tests/ansible/roles/install_node/tasks/main.yml
@@ -28,7 +28,7 @@
     group: root
 
 - name: Run boostrap script
-  shell: /root/ecs/bootstrap.sh -n -b http://10.1.83.5/alpine -g -k /root/ecs/contrib/sslproxycert/emc_ssl.pem -p 10.1.83.5:3128 -c /root/ecs/deploy.yml
+  shell: /root/ecs/bootstrap.sh -n --build-from http://10.1.83.5/alpine --vm-tools --proxy-cert /root/ecs/contrib/sslproxycert/emc_ssl.pem --proxy-endpoint 10.1.83.5:3128 -c /root/ecs/deploy.yml --centos-mirror 10.1.83.5
 
 - name: reboot nodes
   shell: sleep 2 && shutdown -r now "Ansible reboot"

diff --git a/tests/ansible/templates/deploy.yml.j2 b/tests/ansible/templates/deploy.yml.j2
@@ -10,6 +10,8 @@ facts:
   ssh_defaults:
     ssh_username: {{ ansible_ssh_user }}
     ssh_password: {{ ansible_ssh_pass }}
+    ansible_become_pass: {{ ansible_ssh_pass }}
+    ssh_crypto: rsa
 
   node_defaults:
     dns_domain: local

diff --git a/ui/__init__.py b/ui/__init__.py
@@ -1,5 +1,5 @@
 # coding=utf-8
 import tui
-from tui.defaults import *
+from tui.constants import *
 import ui
 
diff --git a/ui/ansible/clicmd_access.yml b/ui/ansible/clicmd_access.yml
@@ -3,7 +3,8 @@
   gather_facts: false
   vars:
     ansible_ssh_pipelining: false
-    ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
+  vars_files:
+    - vars/{{auth_type}}_auth.yml
   tasks:
     - ping: "{{inventory_hostname}}"
 
@@ -12,7 +13,8 @@
   gather_facts: true
   vars:
     ansible_ssh_pipelining: false
-    ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
+  vars_files:
+    - vars/{{auth_type}}_auth.yml
   tasks:
      - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }}
 

diff --git a/ui/ansible/clicmd_access_host.yml b/ui/ansible/clicmd_access_host.yml
@@ -2,7 +2,8 @@
   hosts: install_node
   vars:
     ansible_ssh_pipelining: false
-    ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
+  vars_files:
+    - vars/{{auth_type}}_auth.yml
   tasks:
      - group_by: key=os_{{ ansible_distribution }}_{{ ansible_distribution_major_version }}
 
@@ -15,6 +16,7 @@
   hosts: os_CentOS_7
   vars:
     ansible_ssh_pipelining: false
-    ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
+  vars_files:
+    - vars/{{auth_type}}_auth.yml
   roles:
     - CentOS_7_configure_ssh
diff --git a/ui/ansible/roles/CentOS_7_configure_ssh/tasks/main.yml b/ui/ansible/roles/CentOS_7_configure_ssh/tasks/main.yml
@@ -1,17 +1,23 @@
 #- name: Set selinux permissive
 #  selinux: policy=targeted state=permissive
 
+#- debug:
+#    msg: "keys_ed25519.enabled == true"
+#  when:
+
 - name: CentOS 7 | Distribute ed25519 ssh key
   authorized_key:
     state: present
     user: "{{ ansible_user }}"
     key: "{{ lookup('file', '{{ssh_dir}}/id_ed25519.pub') }}"
+  when: keys_ed25519.enabled == true
 
 - name: CentOS 7 | Distribute rsa ssh key
   authorized_key:
     state: present
     user: "{{ ansible_user }}"
     key: "{{ lookup('file', '{{ssh_dir}}/id_rsa.pub') }}"
+  when: keys_rsa.enabled == true
 
 # old style
 #  authorized_key: state=present user="{{ ansible_user }}" key="{{ lookup('file', '{{ansible_env.HOME}}/.ssh/id_rsa.pub') }}"

diff --git a/ui/ansible/roles/installer_generate_ssh_keys/tasks/main.yml b/ui/ansible/roles/installer_generate_ssh_keys/tasks/main.yml
@@ -1,20 +1,25 @@
-- include_vars: main.yml
-
 - name: Installer | Check RSA Keypair
   stat: path="{{keys_rsa.priv_key}}"
   register: rsa_keypair
+  when: keys_rsa.enabled == True
 
 - name: Installer | Generate RSA Keypair
   command: "{{keys_rsa.keygen_cmd}} {{keys_rsa.priv_key}}"
-  when: not rsa_keypair.stat.exists
+  when: keys_rsa.enabled == True and ( not rsa_keypair.stat.exists )
 
 - name: Installer | Check ed25519 Keypair
   stat: path="{{keys_ed25519.priv_key}}"
   register: ed25519_keypair
+  when: keys_ed25519.enabled
 
 - name: Installer | Generate ed25519 Keypair
   command: "{{keys_ed25519.keygen_cmd}} {{keys_ed25519.priv_key}}"
-  when: not ed25519_keypair.stat.exists
+  when: keys_ed25519.enabled and ( not ed25519_keypair.stat.exists )
+
+- name: Installer | Fail when no crypto selected
+  fail:
+    msg: "No ssh pubkey auth crypto type selected. Check ssh_defaults/ssh_crypto in deploy.yml."
+  when: ( not keys_rsa.enabled ) and ( not keys_ed25519.enabled )
 
 - name: Installer | Ensure directory permissions on ssh keystore
   file:

diff --git a/ui/ansible/roles/installer_generate_ssh_keys/vars/main.yml b/ui/ansible/roles/installer_generate_ssh_keys/vars/main.yml
@@ -1,24 +1,24 @@
-keys_rsa:
-  keygen_cmd: ssh-keygen -N '' -t rsa -b 4096 -o -a 100 -f
-  priv_key: /opt/ssh/id_rsa
-  pub_key: /opt/ssh/id_rsa.pub
-keys_ed25519:
-  keygen_cmd: ssh-keygen -N '' -t ed25519 -o -a 100 -f
-  priv_key: /opt/ssh/id_ed25519
-  pub_key: /opt/ssh/id_ed25519.pub
-ssh_dir_paths:
-  /opt/ssh:
-    mode: 700
-    owner: root
-    group: root
-    state: directory
-  /opt/ssh/id_ed25519:
-    mode: 600
-    owner: root
-    group: root
-    state: file
-  /opt/ssh/id_rsa:
-    mode: 600
-    owner: root
-    group: root
-    state: file
+#keys_rsa:
+#  keygen_cmd: ssh-keygen -N '' -t rsa -b 4096 -o -a 100 -f
+#  priv_key: /opt/ssh/id_rsa
+#  pub_key: /opt/ssh/id_rsa.pub
+#keys_ed25519:
+#  keygen_cmd: ssh-keygen -N '' -t ed25519 -o -a 100 -f
+#  priv_key: /opt/ssh/id_ed25519
+#  pub_key: /opt/ssh/id_ed25519.pub
+#ssh_dir_paths:
+#  /opt/ssh:
+#    mode: 700
+#    owner: root
+#    group: root
+#    state: directory
+#  /opt/ssh/id_ed25519:
+#    mode: 600
+#    owner: root
+#    group: root
+#    state: file
+#  /opt/ssh/id_rsa:
+#    mode: 600
+#    owner: root
+#    group: root
+#    state: file
diff --git a/ui/ansible/templates/all.j2 b/ui/ansible/templates/all.j2
@@ -1,4 +1,7 @@
 # DANGER: Rendered by installer - direct edits WILL be overwritten when ecsdeploy next runs!
+#
+### From etc/config.yml
+#
 host_root_dir: {{ ui.host_root_dir }}
 cache_dir: {{ ui.cache_dir }}
 host_cache_dir: {{ ui.host_cache_dir }}
@@ -16,6 +19,42 @@ product_flavor: {{ product.flavor }}
 product_common_name: {{ product.common_name }}
 ffx_sem: {{ ui.ffx_sem }}
 #
+### SSH Public Key Auth
+#
+auth_type: {{ 'pubkey' if facts.ssh_defaults.ssh_password == facts.ssh_defaults.ssh_username else 'password' }}
+keys_rsa:
+  keygen_cmd: ssh-keygen -N '' -t rsa -b 4096 -o -a 100 -f
+  priv_key: /opt/ssh/{{ facts.ssh_defaults.ssh_private_key if facts.ssh_defaults.ssh_private_key is defined and facts.ssh_defaults.ssh_crypto == 'rsa' else 'id_rsa' }}
+  pub_key: /opt/ssh/{{ facts.ssh_defaults.ssh_public_key if facts.ssh_defaults.ssh_public_key is defined and facts.ssh_defaults.ssh_crypto == 'rsa' else 'id_rsa.pub' }}
+  enabled: {{ 'true' if facts.ssh_defaults.ssh_crypto == 'rsa' else 'false' }}
+keys_ed25519:
+  keygen_cmd: ssh-keygen -N '' -t ed25519 -o -a 100 -f
+  priv_key: /opt/ssh/{{ facts.ssh_defaults.ssh_private_key if facts.ssh_defaults.ssh_private_key is defined and facts.ssh_defaults.ssh_crypto == 'ed25519' else 'ed25519' }}
+  pub_key: /opt/ssh/{{ facts.ssh_defaults.ssh_public_key if facts.ssh_defaults.ssh_public_key is defined and facts.ssh_defaults.ssh_crypto == 'ed25519' else 'ed25519.pub' }}
+  enabled: {{ 'true' if facts.ssh_defaults.ssh_crypto == 'ed25519' else 'false' }}
+ssh_dir_paths:
+  {{ ui.ssh_dir }}:
+    mode: 700
+    owner: root
+    group: root
+    state: directory
+{% if facts.ssh_defaults.ssh_crypto == 'ed25519' %}
+  {{ ui.ssh_dir }}/{{ facts.ssh_defaults.ssh_private_key if facts.ssh_defaults.ssh_private_key is defined and facts.ssh_defaults.ssh_crypto == 'ed25519' else 'ed25519' }}:
+    mode: 600
+    owner: root
+    group: root
+    state: file
+{% endif %}
+{% if facts.ssh_defaults.ssh_crypto == 'rsa' %}
+  {{ ui.ssh_dir }}/{{ facts.ssh_defaults.ssh_private_key if facts.ssh_defaults.ssh_private_key is defined and facts.ssh_defaults.ssh_crypto == 'rsa' else 'id_rsa' }}:
+    mode: 600
+    owner: root
+    group: root
+    state: file
+{% endif %}
+#
+### Required Firewall Ports
+#
 firewall_ports_service_clients:
 # CAS API
   - 3218/tcp
@@ -33,6 +72,5 @@ firewall_ports_management_clients:
 # Management API
   - 4443/tcp
 #
-
-#
 #
+# AUX below
diff --git a/ui/ansible/vars/password_auth.yml b/ui/ansible/vars/password_auth.yml
@@ -0,0 +1 @@
+ansible_ssh_common_args: -o PreferredAuthentications=password -o PubkeyAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
diff --git a/ui/ansible/vars/pubkey_auth.yml b/ui/ansible/vars/pubkey_auth.yml
@@ -0,0 +1 @@
+ansible_ssh_common_args: -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
diff --git a/ui/ecsconfig.py b/ui/ecsconfig.py
@@ -26,7 +26,7 @@
 import time
 import simplejson
 from sarge import Capture, run, shell_format, capture_both, get_both
-from tui.defaults import *
+from tui.constants import *
 from ecsclient.client import Client
 from ecsclient.common.exceptions import ECSClientException
 
@@ -369,7 +369,7 @@ def get_license():
         return conf.api_client.licensing.get_license()['license_text']
 
     def add_license(license_blob):
-        # license_text is a global variable from defaults.py which
+        # license_text is a global variable from constants.py which
         # can be overridden.
         # o(license_blob)
         #license_dict = {"license_text": license_blob.rstrip('\n')}