Merge pull request elastic#379 from gingerwizard/master

Fix for issue elastic#369

.kitchen.yml

@@ -116,4 +116,4 @@ suites:
     attributes:
     provisioner:
       playbook: test/integration/issue-test.yml
-      idempotency_test: true
+      idempotency_test: false

filter_plugins/custom.py

@@ -21,30 +21,37 @@ def append_to_list(values=[], suffix=''):
 def array_to_str(values=[],separator=','):
     return separator.join(values)
 
-def extract_role_users(users={}):
+def extract_role_users(users={},exclude_users=[]):
     role_users=[]
     for user,details in users.iteritems():
-        if "roles" in details:
+        if user not in exclude_users and "roles" in details:
             for role in details["roles"]:
                 role_users.append(role+":"+user)
     return role_users
 
 def filename(filename=''):
     return os.path.splitext(os.path.basename(filename))[0]
 
-def filter_reserved(user_roles={}):
+def remove_reserved(user_roles={}):
     not_reserved = []
     for user_role,details in user_roles.items():
         if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]:
             not_reserved.append(user_role)
     return not_reserved
 
+def filter_reserved(users_role={}):
+    reserved = []
+    for user_role,details in users_role.items():
+        if "metadata" in details and "_reserved" in details["metadata"] and details["metadata"]["_reserved"]:
+            reserved.append(user_role)
+    return reserved
 
 class FilterModule(object):
     def filters(self):
         return {'modify_list': modify_list,
         'append_to_list':append_to_list,
+        'filter_reserved':filter_reserved,
         'array_to_str':array_to_str,
         'extract_role_users':extract_role_users,
-        'filter_reserved':filter_reserved,
+        'remove_reserved':remove_reserved,
         'filename':filename}

tasks/elasticsearch-parameters.yml

@@ -23,6 +23,13 @@
 - fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
   when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
 
+- set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }}
+  when: es_users is defined and es_users.file is defined and (es_users.file.keys() | length > 0) and (es_users.file.keys() | intersect (reserved_xpack_users) | length > 0)
+
+- fail:
+     msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM."
+  when: file_reserved_users | default([]) | length > 0
+
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
 - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}

tasks/elasticsearch-template.yml

@@ -8,15 +8,6 @@
   with_fileglob:
     - "{{ es_templates_fileglob | default('') }}"
 
-
-- name: Ensure elasticsearch is started
-  service: name={{instance_init_script | basename}} state=started enabled=yes
-  when: es_start_service and load_templates.changed
-
-- name: Wait for elasticsearch to startup
-  wait_for: host={{es_api_host}} port={{es_api_port}} delay=10
-  when: es_start_service and load_templates.changed
-
 - name: Install templates without auth
   uri:
     url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"

tasks/main.yml

@@ -39,12 +39,6 @@
 
 - meta: flush_handlers
 
-#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
-- include: elasticsearch-template.yml
-  when: es_templates
-  tags:
-      - templates
-
 - name: Make sure elasticsearch is started
   service: name={{instance_init_script | basename}} state=started enabled=yes
   when: es_start_service
@@ -53,10 +47,27 @@
   wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1
   when: es_restarted is defined and es_restarted.changed and es_start_service
 
+- set_fact: manage_native_realm=false
+
+- set_fact: manage_native_realm=true
+  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+
+# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
+- name: Wait 15 seconds for the Native Relm to come up
+  pause: seconds=15
+  when: manage_native_realm
+
 - name: activate-license
   include: ./xpack/security/elasticsearch-xpack-activation.yml
   when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
 
 #perform security actions here now elasticsearch is started
 - include: ./xpack/security/elasticsearch-security-native.yml
-  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+  when: manage_native_realm
+
+#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
+#We also do after the native realm to ensure any changes are applied here first and its denf up.
+- include: elasticsearch-template.yml
+  when: es_templates
+  tags:
+      - templates

tasks/xpack/security/elasticsearch-security-file.yml

@@ -1,5 +1,5 @@
 ---
-- set_fact: manage_file_users=es_users is defined and es_users.file is defined
+- set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0
 
 #List current users
 - name: List Users
@@ -16,21 +16,20 @@
   command: >
     {{es_home}}/bin/x-pack/users userdel {{item}}
   with_items: "{{users_to_remove | default([])}}"
-  when: manage_file_users and (users_to_remove | length > 0)
+  when: manage_file_users
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
-
 - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }}
   when: manage_file_users
 
 #Add users
 - name: Add Users
   command: >
     {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}}
-  with_items: "{{users_to_add | default([])}}"
-  when: manage_file_users and users_to_add | length > 0
+  with_items: "{{ users_to_add | default([]) }}"
+  when: manage_file_users
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
@@ -39,17 +38,17 @@
 #Set passwords for all users declared - Required as the useradd will not change existing user passwords
 - name: Set User Passwords
   command: >
-    {{es_home}}/bin/x-pack/users passwd {{item.key}} -p {{item.value.password}}
-  with_dict: "{{(es_users | default({'file':{}})).file}}"
-  when: manage_file_users and es_users.file.keys() | length > 0
+    {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}}
+  with_items: "{{ es_users.file.keys() | default([]) }}"
+  when: manage_file_users
   #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
   changed_when: False
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
-- set_fact: users_roles={{es_users.file | extract_role_users}}
+- set_fact: users_roles={{es_users.file | extract_role_users () }}
   when: manage_file_users
 
 #Copy Roles files

tasks/xpack/security/elasticsearch-security-native.yml

@@ -1,18 +1,15 @@
 ---
+- set_fact: change_api_password=false
 
 - set_fact: manage_native_users=false
 
 - set_fact: manage_native_users=true
-  when: es_users is defined and es_users.native is defined
+  when: es_users is defined and es_users.native is defined and es_users.native.keys() | length > 0
 
 - set_fact: manage_native_roles=false
 
 - set_fact: manage_native_roles=true
-  when: es_roles is defined and es_roles.native is defined
-
-# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
-- name: Wait 15 seconds for the Native Relm to come up
-  pause: seconds=15
+  when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | length > 0
 
 #If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load
 
@@ -28,15 +25,40 @@
   register: user_list_response
   when: manage_native_users
 
+- set_fact: reserved_users={{ user_list_response.json | filter_reserved }}
+  when: manage_native_users
+
 #Current users not inc. those reserved
-- set_fact: current_users={{ user_list_response.json | filter_reserved }}
+- set_fact: current_users={{ user_list_response.json.keys() | difference (reserved_users) }}
   when: manage_native_users
 
-#Identify non declared users
-- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }}
+#We are changing the es_api_basic_auth_username password, so we need to do it first and update the param
+- set_fact: native_users={{ es_users.native }}
   when: manage_native_users
 
-#Delete all non required users
+- set_fact: change_api_password=true
+  when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined
+
+- name: Update API User Password
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password
+    method: POST
+    body_format: json
+    body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: change_api_password
+
+- set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}}
+  when: change_api_password
+
+#Identify users that are present in ES but not declared and thus should be removed
+- set_fact: users_to_remove={{ current_users | difference ( native_users.keys() ) }}
+  when: manage_native_users
+
+#Delete all non required users NOT inc. reserved
 - name: Delete Native Users
   uri:
     url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
@@ -45,26 +67,50 @@
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_users and users_to_remove | length > 0
-  with_items: "{{users_to_remove | default([]) }}"
+  when: manage_native_users
+  with_items: "{{ users_to_remove | default([]) }}"
 
-- set_fact: native_users={{ es_users.native }}
-  when: manage_native_users and es_users.native.keys() > 0
+- set_fact: users_to_ignore={{ native_users.keys() | intersect (reserved_users) }}
+  when: manage_native_users
+
+- debug:
+    msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}"
+  when: manage_native_users and users_to_ignore | length > 0
+
+#Update password on all reserved users
+- name: Update Reserved User Passwords
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password
+    method: POST
+    body_format: json
+    body: "{ \"password\":\"{{native_users[item].password}}\" }"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: native_users[item].password is defined
+  no_log: True
+  with_items: "{{ users_to_ignore | default([]) }}"
 
-#Overwrite all other users
-- name: Update Native Users
+- set_fact: users_to_modify={{ native_users.keys() | difference (reserved_users) }}
+  when: manage_native_users
+
+#Overwrite all other users NOT inc. those reserved
+- name: Update Non-Reserved Native User Details
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
     method: POST
     body_format: json
-    body: "{{item.value | to_json}}"
+    body: "{{ native_users[item] | to_json }}"
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_users and native_users.keys() > 0
+  when: manage_native_users
   no_log: True
-  with_dict: "{{native_users | default({}) }}"
+  with_items: "{{ users_to_modify | default([]) }}"
+
+## ROLE CHANGES
 
 #List current roles not. inc those reserved
 - name: List Native Roles
@@ -79,16 +125,23 @@
   register: role_list_response
   when: manage_native_roles
 
-- set_fact: current_roles={{ role_list_response.json | filter_reserved }}
+- set_fact: reserved_roles={{ role_list_response.json | filter_reserved }}
   when: manage_native_roles
 
-- debug: msg="{{current_roles}}"
+- set_fact: current_roles={{ role_list_response.json.keys() | difference (reserved_roles) }}
   when: manage_native_roles
 
+- set_fact: roles_to_ignore={{ es_roles.native.keys() | intersect (reserved_roles) | default([]) }}
+  when: manage_native_roles
+
+- debug:
+    msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}"
+  when: manage_native_roles and roles_to_ignore | length > 0
+
 - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys()  ) }}
   when: manage_native_roles
 
-#Delete all non required roles
+#Delete all non required roles NOT inc. reserved
 - name: Delete Native Roles
   uri:
     url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
@@ -97,23 +150,22 @@
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_roles and roles_to_remove | length > 0
+  when: manage_native_roles
   with_items: "{{roles_to_remove | default([]) }}"
 
+- set_fact: roles_to_modify={{ es_roles.native.keys() | difference (reserved_roles) }}
+  when: manage_native_roles
 
-- set_fact: native_roles={{ es_roles.native }}
-  when: manage_native_roles and es_roles.native.keys() > 0
-
-#Update other roles
+#Update other roles - NOT inc. reserved roles
 - name: Update Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
     method: POST
     body_format: json
-    body: "{{item.value | to_json}}"
+    body: "{{ es_roles.native[item] | to_json}}"
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_roles and native_roles.keys() > 0
-  with_dict: "{{ native_roles | default({})}}"
+  when: manage_native_roles
+  with_items: "{{ roles_to_modify | default([]) }}"