Merge pull request elastic#269 from gingerwizard/master

Support for plugin diffs + improved testing on minor upgrades + fixes

.kitchen.yml

@@ -12,7 +12,6 @@ provisioner:
   http_proxy: <%= ENV['HTTP_PROXY'] %>
   https_proxy: <%= ENV['HTTPS_PROXY'] %>
   no_proxy: localhost,127.0.0.1
-  idempotency_test: true
 
 platforms:
   - name: ubuntu-14.04
@@ -21,7 +20,8 @@ platforms:
       privileged: true
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
-        - apt-get update && apt-get -y -q install python-apt python-pycurl
+        - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip
+        - pip install jmespath
       use_sudo: false
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
   - name: ubuntu-16.04
@@ -31,7 +31,8 @@ platforms:
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
         - apt-get install -y -q net-tools
-        - apt-get update && apt-get -y -q install python-apt python-pycurl
+        - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip
+        - pip install jmespath
       use_sudo: false
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
       run_command: "/sbin/init"
@@ -46,6 +47,7 @@ platforms:
         - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config
+        - pip install jmespath
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
       use_sudo: false
       run_command: "/sbin/init"
@@ -59,6 +61,7 @@ platforms:
         - rm /etc/yum.repos.d/epel*repo /etc/yum.repos.d/puppetlabs-pc1.repo
         - yum -y install initscripts
         - yum clean all
+        - pip install jmespath
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
       run_command: "/usr/sbin/init"
       privileged: true
@@ -67,15 +70,13 @@ platforms:
 suites:
   - name: standard-5x
     provisioner:
+      idempotency_test: true
       playbook: test/integration/standard.yml
     run_list:
     attributes:
   - name: package-5x
     run_list:
     attributes:
-      extra_vars:
-        es_plugins:
-          - plugin: ingest-geoip
     provisioner:
       playbook: test/integration/package.yml
   - name: config-5x
@@ -86,13 +87,11 @@ suites:
   - name: multi-5x
     run_list:
     attributes:
-      extra_vars:
-        es_plugins:
-          - plugin: ingest-geoip
     provisioner:
       playbook: test/integration/multi.yml
+      idempotency_test: true
   - name: xpack-5x
     run_list:
     attributes:
     provisioner:
-      playbook: test/integration/xpack.yml
+      playbook: test/integration/xpack.yml

defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 es_major_version: "5.x"
-es_version: "5.1.2"
+es_version: "5.2.2"
 es_version_lock: false
 es_use_repository: true
 es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch"

tasks/elasticsearch-config.yml

@@ -47,7 +47,6 @@
   - restart elasticsearch
 
 #Copy the logging.yml
-
 - name: Copy log4j2.properties File for Instance
   template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch

tasks/elasticsearch-plugins.yml

@@ -5,25 +5,35 @@
 - set_fact: es_plugins_reinstall=true
   when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none
 
-- set_fact: list_command="list"
+- set_fact: list_command=""
+#If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent.
+- set_fact: list_command="| grep -vE 'x-pack'"
+  when: not es_plugins_reinstall
 
-#List currently installed plugins - ignore xpack if > v 2.0
+#List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version.
 - name: Check installed elasticsearch plugins
-  shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack'"
+  shell: "ls {{es_home}}/plugins {{list_command}}"
   register: installed_plugins
-  failed_when: "'ERROR' in installed_plugins.stdout"
   changed_when: False
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
+#if es_plugins_reinstall is set to true we remove ALL plugins
+- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}"
+  when: es_plugins_reinstall
+
+#if the plugins listed are different than those requested, we remove those installed but not listed in the config
+- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}"
+  when: not es_plugins_reinstall
+
 # This removes any currently installed plugins (to prevent errors when reinstalling)
 - name: Remove elasticsearch plugins
   command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent"
   ignore_errors: yes
-  with_items: "{{ installed_plugins.stdout_lines | default([]) }}"
-  when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0]
+  with_items: "{{ plugins_to_remove | default([]) }}"
+  when: es_plugins_reinstall and plugins_to_remove | length > 0
   notify: restart elasticsearch
   register: plugin_removed
   environment:

tasks/elasticsearch-scripts.yml

@@ -10,7 +10,7 @@
       - always
 
 - name: Create script dir
-  file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}
+  file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes
 
 - name: Copy default scripts to elasticsearch
   copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}

test/integration/config-5x/serverspec/default_spec.rb

@@ -1,6 +1,6 @@
 require 'config_spec'
 
 describe 'Config Tests v 5.x' do
-  include_examples 'config::init', "5.1.2"
+  include_examples 'config::init', "5.2.2", ["ingest-attachment","ingest-user-agent"]
 end
 

test/integration/config.yml

@@ -1,13 +1,31 @@
 ---
 #Test explicit setting of parameters and variables
-- name: Elasticsearch Config tests
+- name: Elasticsearch Config initial
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true } }
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: false } }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
     es_heap_size: 1g
-    es_api_port: 9201
+    es_api_port: 9201
+    es_plugins:
+      - plugin: ingest-geoip
+
+#Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed.
+- name: Elasticsearch Config test modify
+  hosts: localhost
+  roles:
+    #expand to all available parameters
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: true } }
+  vars:
+    es_scripts: false
+    es_templates: false
+    es_version_lock: false
+    es_heap_size: 1g
+    es_api_port: 9401
+    es_plugins:
+      - plugin: ingest-attachment
+      - plugin: ingest-user-agent

test/integration/helpers/serverspec/config_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-shared_examples 'config::init' do |es_version|
+shared_examples 'config::init' do |es_version,plugins|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -28,14 +28,14 @@
 
   #test configuration parameters have been set - test all appropriately set in config file
   describe file('/etc/elasticsearch/node1/elasticsearch.yml') do
-    it { should contain 'http.port: 9201' }
-    it { should contain 'transport.tcp.port: 9301' }
-    it { should contain 'node.data: false' }
+    it { should contain 'http.port: 9401' }
+    it { should contain 'transport.tcp.port: 9501' }
+    it { should contain 'node.data: true' }
     it { should contain 'node.master: true' }
     it { should contain 'cluster.name: custom-cluster' }
     it { should contain 'node.name: node1' }
     it { should contain 'bootstrap.memory_lock: true' }
-    it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9301' }
+    it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' }
     it { should contain 'path.conf: /etc/elasticsearch/node1' }
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
     it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' }
@@ -63,27 +63,47 @@
   end
 
   #test we started on the correct port was used
-  describe command('curl -s "localhost:9201"') do
+  describe command('curl -s "localhost:9401"') do
     #TODO: This is returning an empty string
     #its(:stdout) { should match /\"status\" : 200/ }
     its(:exit_status) { should eq 0 }
   end
 
   #test to make sure mlock was applied
-  describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep mlockall') do
+  describe command('curl -s "localhost:9401/_nodes/process?pretty" | grep mlockall') do
     its(:stdout) { should match /true/ }
     its(:exit_status) { should eq 0 }
   end
 
 
   describe 'version check' do
     it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9201 | grep number')
+      command = command('curl -s localhost:9401 | grep number')
       expect(command.stdout).to match(es_version)
       expect(command.exit_status).to eq(0)
     end
   end
 
+  for plugin in plugins
+    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+    #confirm plugins are installed and the correct version
+    describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+      its(:exit_status) { should eq 0 }
+    end
+  end
+
+  #explit test to make sure ingest-geoip is not installed
+  describe file('/usr/share/elasticsearch/plugins/ingest-geoip') do
+    it { should_not exist }
+  end
+  #confirm plugins are installed and the correct version
+  describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+es_version+'"\'') do
+    its(:exit_status) { should eq 1 }
+  end
+
   describe file('/etc/init.d/elasticsearch') do
     it { should_not exist }
   end

test/integration/helpers/serverspec/package_spec.rb

@@ -74,12 +74,13 @@
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
-
-    describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
+    #confirm plugins are installed and the correct version
+    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
 
+
   describe file('/etc/init.d/elasticsearch') do
     it { should_not exist }
   end

test/integration/helpers/serverspec/xpack_spec.rb

@@ -40,7 +40,7 @@
 
   describe 'version check' do
     it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9200 -u es_admin:changeMe | grep number')
+      command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
       expect(command.stdout).to match(es_version)
       expect(command.exit_status).to eq(0)
     end
@@ -79,7 +79,7 @@
   #Test if x-pack is activated
   describe 'x-pack activation' do
     it 'should be activated and valid' do
-      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMe')
+      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain')
       expect(command.stdout).to match('"status" : "active"')
       expect(command.exit_status).to eq(0)
     end
@@ -90,7 +90,7 @@
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep x-pack') do
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
     its(:exit_status) { should eq 0 }
   end
 
@@ -110,7 +110,7 @@
       it { should be_owned_by 'elasticsearch' }
     end
 
-    describe command('curl -s -u es_admin:changeMe localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
+    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
@@ -137,19 +137,11 @@
 
 
   #Test native roles and users are loaded
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 243b362bd47623c0b91a1fafbce2b6f5') do
+  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep 74bcc9f9534b253c1204e264df21496c') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"elastic\":{\"username\":\"elastic\",\"roles\":\[\"superuser\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana\":{\"username\":\"kibana\",\"roles\":\[\"kibana\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{},\"enabled\":true}}"') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"superuser\":{\"cluster\":\[\"all\"\],\"indices\":\[{\"names\":\[\"\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\"\*\"\],\"metadata\":{\"_reserved\":true}},\"transport_client\":{\"cluster\":\[\"transport_client\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"kibana_user\":{\"cluster\":\[\"monitor\"\],\"indices\":\[{\"names\":\[\".kibana\*\"\],\"privileges\":\[\"manage\",\"read\",\"index\",\"delete\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"monitoring_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\".monitoring-\*\"\],\"privileges\":\[\"read\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"remote_monitoring_agent\":{\"cluster\":\[\"manage_index_templates\",\"manage_ingest_pipelines\",\"monitor\"\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\"\.monitoring-\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"ingest_admin\":{\"cluster\":\[\"manage_index_templates\",\"manage_pipeline\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"reporting_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.reporting-\*\"\],\"privileges\":\[\"read\",\"write\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\],\"metadata\":{}}}"') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 78a0696c9c9690042cec2c1f16860cfc') do
+  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 2bf3ffbb9cabf26bb25de6334c4da323') do
     its(:exit_status) { should eq 0 }
   end
 
@@ -165,7 +157,7 @@
 
   describe 'Template Installed' do
     it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe')
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain')
       expect(command.stdout).to match(/basic/)
       expect(command.exit_status).to eq(0)
     end
@@ -174,7 +166,7 @@
   #This is possibly subject to format changes in the response across versions so may fail in the future
   describe 'Template Contents Correct' do
     it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe | md5sum')
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
       expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
     end
   end

test/integration/multi-5x/serverspec/default_spec.rb

@@ -2,7 +2,7 @@
 
 
 describe 'Multi Tests v 5.x' do
-  include_examples 'multi::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'multi::init', "5.2.2", ["ingest-geoip"]
 end
 
 

test/integration/multi.yml

@@ -7,6 +7,8 @@
     es_templates: true
     es_heap_size: "1g"
     es_api_port: 9200
+    es_plugins:
+      - plugin: ingest-geoip
   roles:
     - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } }
 
@@ -17,6 +19,8 @@
     es_templates: true
     es_heap_size: "1g"
     es_api_port: 9201
+    es_plugins:
+      - plugin: ingest-geoip
   roles:
     - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } }
 #Plugins installed for this test are specified in .kitchen.yml under suite

test/integration/package-5x/serverspec/default_spec.rb

@@ -2,5 +2,5 @@
 
 
 describe 'Package Tests v 5.x' do
-  include_examples 'package::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'package::init', "5.2.2", ["ingest-attachment","ingest-geoip"]
 end