METRON-939: Upgrade ElasticSearch and Kibana

dependencies_with_url.csv

@@ -310,6 +310,29 @@ org.springframework.security.kerberos:spring-security-kerberos-core:jar:1.0.1.RE
 org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka
 ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java
 org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j
+com.github.spullara.mustache.java:compiler:jar:0.9.3:compile,ASLv2,https://github.com/spullara/mustache.java/blob/master/LICENSE
+io.netty:netty-buffer:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
+io.netty:netty-codec-http:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
+io.netty:netty-codec:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
+io.netty:netty-common:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
+io.netty:netty-handler:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
+io.netty:netty:jar:3.10.6.Final:compile,ASLv2,http://netty.io/
+io.netty:netty-resolver:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
+io.netty:netty-transport:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
+joda-time:joda-time:jar:2.9.5:compile,ASLv2,https://github.com/JodaOrg/joda-time
+net.sf.jopt-simple:jopt-simple:jar:5.0.2:compile,The MIT License,http://jopt-simple.sourceforge.net
+org.elasticsearch.client:elasticsearch-rest-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.client:transport:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch:elasticsearch:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch:jna:jar:4.4.0-1:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.plugin:lang-mustache-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.plugin:parent-join-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.plugin:percolator-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.plugin:reindex-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.plugin:transport-netty3-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.plugin:transport-netty4-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch:securesm:jar:1.1:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.hdrhistogram:HdrHistogram:jar:2.1.9:compile,BSD,https://github.com/HdrHistogram/HdrHistogram/blob/master/LICENSE.txt
 com.trendmicro:tlsh:jar:3.7.1:compile,ASLv2,https://github.com/trendmicro/tlsh
 org.glassfish:javax.json:jar:1.0.4:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/jsonp
 org.eclipse.persistence:javax.persistence:jar:2.1.1:compile,EPL 1.0,http://www.eclipse.org/eclipselink

metron-analytics/metron-profiler-client/pom.xml

@@ -139,6 +139,14 @@
             <version>${global_storm_version}</version>
             <scope>provided</scope>
             <exclusions>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-core</artifactId>
+                </exclusion>
                 <exclusion>
                     <artifactId>servlet-api</artifactId>
                     <groupId>javax.servlet</groupId>

metron-analytics/metron-profiler-common/pom.xml

@@ -114,6 +114,14 @@
             <version>${global_storm_version}</version>
             <scope>provided</scope>
             <exclusions>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-core</artifactId>
+                </exclusion>
                 <exclusion>
                     <artifactId>log4j-slf4j-impl</artifactId>
                     <groupId>org.apache.logging.log4j</groupId>

metron-analytics/metron-profiler/pom.xml

@@ -202,6 +202,14 @@
             <artifactId>storm-core</artifactId>
             <version>${global_storm_version}</version>
             <exclusions>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-core</artifactId>
+                </exclusion>
                 <exclusion>
                     <artifactId>servlet-api</artifactId>
                     <groupId>javax.servlet</groupId>
@@ -221,6 +229,18 @@
             </exclusions>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <version>${global_log4j_core_version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>${global_log4j_core_version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka_2.10</artifactId>

metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md

@@ -458,7 +458,7 @@ Client
 
 - Kibana:
     * Set "kibana_es_url" to `http://<replace_with_elasticsearch_master_hostname>:9200`. "replace_with_elasticsearch_master_hostname" is the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.
-    * Change kibana_default_application to "dashboard/Metron-Dashboard"
+    * Change kibana_default_application to "dashboard/AV-YpDmwdXwc6Ua9Muh9"
 
 - Metron:
     Set "Elasticsearch Hosts" to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.

metron-deployment/packaging/ambari/README.md

@@ -27,6 +27,7 @@ limitations under the License.
 * [Configuration involving dependency services](#configuration-involving-dependency-services)
 * [Kerberos](#kerberos)
 * [Best practices](#best-practices)
+* [Upgrading MPack Services](#upgrading-mpack-services)
 
 ## Overview
 Typically, Ambari Management Pack development will be done in the Vagrant environments. These instructions are specific to Vagrant, but can be adapted for other environemnts (e.g. make sure to be on the correct nodes for server vs agent files)
@@ -464,3 +465,211 @@ This is checked in the indexing master
   * Make sure to `kinit` as the correct user for setting up ACLs in a secured cluster. This is usually kafka for Kafka and hbase for HBase.
   * See `set_hbase_acls` in `METRON.CURRENT/package/scripts/enrichment_commands.py` for an HBase example
   * See `init_kafka_acls` in `METRON.CURRENT/package/scripts/enrichment_commands.py` and  `METRON.CURRENT/package/scripts/metron_service.py` for an Kafka example
+
+## Upgrading MPack Services
+
+Apache Metron currently provides three services as part of its MPack
+* Elasticsearch
+* Kibana
+* Metron
+
+There is currently no mechanism provided for multi-version or backwards compatibility. If you upgrade a service, e.g. Elasticsearch 2.x to 5.x, that is the only version that will be
+supported by Ambari via MPack.
+
+The main steps for upgrading a service are split into add-on and common services for each service within the MPack as follows:
+* Update the common services
+    * Change the service directory to use the new product version number
+    * Update metainfo.xml
+* Update the add-on services
+    * Change the service directory to use the new product version number
+    * Update repoinfo.xml
+    * Update metainfo.xml
+* Update mpack.json
+
+### Update Elasticsearch
+
+#### Update Common Services
+
+1. Change service directory names for Elasticsearch to the new desired version
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}
+    ```
+
+    e.g.
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2
+    ```
+
+1. Update metainfo.xml
+
+   Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g.
+
+   ```
+   <version>5.6.2</version>
+   ...
+   <osSpecifics>
+       <osSpecific>
+           <osFamily>any</osFamily>
+           <packages>
+               <package>
+                   <name>elasticsearch-5.6.2</name>
+               </package>
+           </packages>
+       </osSpecific>
+   </osSpecifics>
+   ```
+
+#### Update Add-on Services
+
+1. Change service directory names for Elasticsearch to the new desired version
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}
+    ```
+
+    e.g.
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2
+    ```
+
+1. Update repoinfo.xml
+
+    See [https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html](https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html) for the latest info.
+
+    Modify the baseurl and repoid in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/repos/repoinfo.xml`, e.g.
+
+    ```
+    <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
+    <repoid>elasticsearch-5.x</repoid>
+    <reponame>ELASTICSEARCH</reponame>
+     ```
+
+1. Update metainfo.xml
+
+   Change the version number in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`.
+   Also make sure to update the "extends" version to point to the updated common-services version, e.g.
+
+   ```
+   <name>ELASTICSEARCH</name>
+   <version>5.6.2</version>
+   <extends>common-services/ELASTICSEARCH/5.6.2</extends>
+   ```
+
+#### Update mpack.json
+
+1. Update the corresponding service_version in the service_versions_map, e.g.
+
+    ```
+    ...
+    "service_versions_map": [
+      {
+        "service_name" : "ELASTICSEARCH",
+        "service_version" : "5.6.2",
+        "applicable_stacks" : [
+            ...
+        ]
+      },
+      ...
+     ]
+    ...
+    ```
+
+### Kibana
+
+**Note:** Curator is included with the Kibana service
+
+#### Update Common Services
+
+1. Change service directory names for Kibana to the new desired version
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}
+    ```
+
+    e.g.
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2
+    ```
+
+1. Update metainfo.xml
+
+   Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g.
+
+   ```
+   <version>5.6.2</version>
+   ...
+   <packages>
+       ...
+       <package>
+           <name>kibana-5.6.2</name>
+       </package>
+   </packages>
+   ```
+
+#### Update Add-on Services
+
+1. Change service directory names for Kibana to the new desired version
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}
+    ```
+
+    e.g.
+
+    ```
+    metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2
+    ```
+
+1. Update repoinfo.xml
+
+    **Note:** for Curator, there is a different repo for rhel 6 vs rhel 7
+
+    See the following links for current repo information for Kibana and Curator.
+    * [https://www.elastic.co/guide/en/kibana/current/rpm.html](https://www.elastic.co/guide/en/kibana/current/rpm.html)
+    * [https://www.elastic.co/guide/en/elasticsearch/client/curator/current/yum-repository.html](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/yum-repository.html)
+
+    Modify the baseurl's and repoid's in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/repos/repoinfo.xml`, e.g.
+
+    ```
+    <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl>
+    <repoid>kibana-5.x</repoid>
+    <reponame>KIBANA</reponame>
+    ...
+    <baseurl>http://packages.elastic.co/curator/5/centos/6</baseurl>
+    <repoid>ES-Curator-5.x</repoid>
+    <reponame>CURATOR</reponame>
+    ```
+
+1. Update metainfo.xml
+
+   Change the version number in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`.
+   Also make sure to update the "extends" version to point to the updated common-services version, e.g.
+   ```
+   <name>KIBANA</name>
+   <version>5.6.2</version>
+   <extends>common-services/KIBANA/5.6.2</extends>
+   ```
+
+#### Update mpack.json
+
+1. Update the corresponding service_version in the service_versions_map, e.g.
+
+    ```
+    ...
+    "service_versions_map": [
+      {
+        "service_name" : "KIBANA",
+        "service_version" : "5.6.2",
+        "applicable_stacks" : [
+            ...
+        ]
+      },
+      ...
+     ]
+    ...
+    ```
+