Add support for xpack

README.md

@@ -13,6 +13,8 @@ From 5.0, this project provides 2 implementations of an elasticsearch Client:
 * The REST client
 * The Transport client (deprecated)
 
+From 6.0, this project supports [X-Pack](https://www.elastic.co/fr/products/x-pack) for official security.
+
 ## Documentation
 
 * For 6.x elasticsearch versions, you are reading the latest documentation.
@@ -50,7 +52,7 @@ Import spring-elasticsearch in you project `pom.xml` file:
 <dependency>
   <groupId>fr.pilato.spring</groupId>
   <artifactId>spring-elasticsearch</artifactId>
-  <version>5.0</version>
+  <version>6.0</version>
 </dependency>
 ```
 
@@ -60,7 +62,7 @@ If you want to set a specific version of the Rest client, add it to your `pom.xm
 <dependency>
     <groupId>org.elasticsearch.client</groupId>
     <artifactId>elasticsearch-rest-client</artifactId>
-    <version>5.6.7</version>
+    <version>6.1.3</version>
 </dependency>
 ```
 
@@ -70,17 +72,41 @@ If you want to use a transport client (deprecated), you must add it to your `pom
 <dependency>
     <groupId>org.elasticsearch.client</groupId>
     <artifactId>transport</artifactId>
-    <version>5.6.7</version>
+    <version>6.1.3</version>
 </dependency>
 ```
 
+If you want to use a transport client secured with X-Pack (deprecated), you must add it to your `pom.xml` file:
+
+```xml
+<dependency>
+    <groupId>org.elasticsearch.client</groupId>
+    <artifactId>x-pack-transport</artifactId>
+    <version>6.1.3</version>
+</dependency>
+```
+
+Note that you'd probably to add also the elastic maven repository:
+
+```xml
+<repositories>
+    <repository>
+        <id>elastic-download-service</id>
+        <name>Elastic Download Service</name>
+        <url>https://artifacts.elastic.co/maven/</url>
+        <releases><enabled>true</enabled></releases>
+        <snapshots><enabled>false</enabled></snapshots>
+    </repository>
+</repositories>
+```
+
 If you want to try out the most recent SNAPSHOT version [deployed on Sonatype](https://oss.sonatype.org/content/repositories/snapshots/fr/pilato/spring/spring-elasticsearch/):
 
 ```xml
 <dependency>
   <groupId>fr.pilato.spring</groupId>
   <artifactId>spring-elasticsearch</artifactId>
-  <version>5.1-SNAPSHOT</version>
+  <version>6.1-SNAPSHOT</version>
 </dependency>
 ```
 
@@ -176,6 +202,18 @@ Better, you should use `@Autowired` annotation.
 @Autowired RestClient client;
 ```
 
+#### Connecting to a secured X-Pack cluster
+
+You need to define the `xpack.security.user` property as follows:
+
+```
+<util:properties id="esProperties">
+    <prop key="xpack.security.user">elastic:changeme</prop>
+</util:properties>
+
+<elasticsearch:rest-client id="esClient" properties="esProperties" />
+```
+
 ### Getting a transport client bean (deprecated)
 
 From 5.0, the Transport Client implementation is deprecated. It is now marked as `optional` so
@@ -261,6 +299,20 @@ You can also add plugins to the transport client in case it needs it:
 <elasticsearch:client id="esClient" plugins="org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin" />
 ```
 
+#### Connecting to a secured X-Pack cluster
+
+You need to define the `xpack.security.user` property as follows:
+
+```
+<util:properties id="esProperties">
+    <prop key="xpack.security.user">elastic:changeme</prop>
+</util:properties>
+
+<elasticsearch:client id="esClient" properties="esProperties" />
+```
+
+Note that it needs that you imported to your project the `x-pack-transport` jar.
+
 ## Automatically create indices
 
 The following examples are documented using the Rest Client implementation `elasticsearch:rest-client` but you can
@@ -492,6 +544,13 @@ public class RestApp {
         public RestClient esClient() throws Exception {
             ElasticsearchRestClientFactoryBean factory = new ElasticsearchRestClientFactoryBean();
             factory.setEsNodes(new String[]{"127.0.0.1:9200"});
+
+            // Begin: If you are running with x-pack
+            Properties props = new Properties();
+            props.setProperty("xpack.security.user", "elastic:changeme");
+		    factory.setProperties(props);
+            // End: If you are running with x-pack
+
             factory.afterPropertiesSet();
             return factory.getObject();
         }
@@ -613,6 +672,28 @@ Special thanks to
 - [Nicolas Labrot](https://github.com/nithril‎) for his contribution about
 [async](https://github.com/dadoonet/spring-elasticsearch/pull/30)
 
+# Running tests
+
+If you want to run tests (integration tests) from your IDE, you need to start first an elasticsearch instance.
+
+If you are not using x-pack, then just run the tests from your IDE. Tests are expecting a node running at `localhost:9200`.
+
+If you are using x-pack, tests are expecting a user named `elastic` with `changeme` as the password.
+You can set this user by running `bin/x-pack/setup-passwords interactive`.
+
+To run the tests using Maven (on the CLI), just run:
+
+```sh
+mvn clean install
+```
+
+Note that when the tests are launched with maven, they are not running with x-pack yet.
+To run tests against x-pack, you need to start elasticsearch with x-pack manually and run the tests with:
+
+```sh
+mvn clean install -Px-pack
+```
+
 # Release guide
 
 To release the project you need to run the release plugin with the `release` profile as you need to sign the artifacts:

pom.xml

@@ -35,6 +35,7 @@
         <skipTests>false</skipTests>
         <skip.unit.tests>${skipTests}</skip.unit.tests>
         <skip.integration.tests>${skipTests}</skip.integration.tests>
+        <tests.cluster.credentials>elastic:changeme</tests.cluster.credentials>
     </properties>
 
     <distributionManagement>
@@ -75,13 +76,24 @@
         <connection>scm:git:git@github.com:dadoonet/spring-elasticsearch.git</connection>
         <developerConnection>scm:git:git@github.com:dadoonet/spring-elasticsearch.git</developerConnection>
       <tag>HEAD</tag>
-  </scm>
+    </scm>
 
     <issueManagement>
         <system>GitHub</system>
         <url>https://github.com/dadoonet/spring-elasticsearch/issues/</url>
     </issueManagement>
 
+    <repositories>
+        <!-- This repository is used to test with x-pack -->
+        <repository>
+            <id>elastic-download-service</id>
+            <name>Elastic Download Service</name>
+            <url>https://artifacts.elastic.co/maven/</url>
+            <releases><enabled>true</enabled></releases>
+            <snapshots><enabled>false</enabled></snapshots>
+        </repository>
+    </repositories>
+
     <dependencies>
         <dependency>
             <groupId>org.hamcrest</groupId>
@@ -133,12 +145,20 @@
             <optional>true</optional>
         </dependency>
 
+        <!-- add the transport client jar as an optional dependency -->
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>transport</artifactId>
             <version>${elasticsearch.version}</version>
             <optional>true</optional>
         </dependency>
+        <!-- add the x-pack jar as an optional dependency -->
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>x-pack-transport</artifactId>
+            <version>${elasticsearch.version}</version>
+            <optional>true</optional>
+        </dependency>
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-client</artifactId>
@@ -369,6 +389,23 @@
     </build>
 
     <profiles>
+        <profile>
+            <id>x-pack</id>
+            <build>
+                <plugins>
+                    <!-- We disable the elasticsearch-maven-plugin as it can not run with x-pack yet -->
+                    <!-- https://github.com/alexcojocaru/elasticsearch-maven-plugin/issues/42 -->
+                    <plugin>
+                        <groupId>com.github.alexcojocaru</groupId>
+                        <artifactId>elasticsearch-maven-plugin</artifactId>
+                        <version>6.4</version>
+                        <configuration>
+                            <skip>true</skip>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
         <profile>
             <id>release</id>
             <build>

src/main/java/fr/pilato/spring/elasticsearch/ElasticsearchAbstractFactoryBean.java

@@ -28,14 +28,16 @@
  * An abstract {@link FactoryBean} used to create an Elasticsearch object.
  * @author David Pilato
  */
-abstract class ElasticsearchAbstractFactoryBean {
+public abstract class ElasticsearchAbstractFactoryBean {
 
     Properties properties;
 
 	boolean async = false;
 
 	ThreadPoolTaskExecutor taskExecutor;
 
+	public final static String XPACK_USER = "xpack.security.user";
+
     /**
      * Elasticsearch properties
      * <p>Example:</p>

src/main/java/fr/pilato/spring/elasticsearch/ElasticsearchRestClientFactoryBean.java

@@ -25,7 +25,12 @@
 import fr.pilato.spring.elasticsearch.proxy.GenericInvocationHandler;
 import fr.pilato.spring.elasticsearch.util.Tuple;
 import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.aop.framework.ProxyFactory;
@@ -352,46 +357,6 @@ private RestClient initialize() throws Exception {
             computeTemplates();
         }
 
-        // TODO ADAPT
-
-        // We extract indexes and mappings to manage from mappings definition
-/*
-		if (mappings != null && mappings.length > 0) {
-            ClusterHealthRequestBuilder healthRequestBuilder = client.admin().cluster().prepareHealth().setWaitForYellowStatus();
-            ClusterStateRequestBuilder clusterStateRequestBuilder = client.admin().cluster().prepareState();
-            Map<String, Collection<String>> indices = getIndexMappings(mappings);
-            for (String index : indices.keySet()) {
-                clusterStateRequestBuilder.setIndices(index);
-            }
-            ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.get();
-
-            boolean checkIndicesStatus = false;
-            for (String index : indices.keySet()) {
-                if (clusterStateResponse.getState().getMetaData().indices().containsKey(index)) {
-                    healthRequestBuilder.setIndices(index);
-                    checkIndicesStatus = true;
-                }
-            }
-
-            if (checkIndicesStatus) {
-                logger.debug("we have to check some indices status as they already exist...");
-                ClusterHealthResponse healths = healthRequestBuilder.get();
-                if (healths.isTimedOut()) {
-                    logger.warn("we got a timeout when checking indices status...");
-                    if (healths.getIndices() != null) {
-                        for (ClusterIndexHealth health : healths.getIndices().values()) {
-                            if (health.getStatus() == ClusterHealthStatus.RED) {
-                                logger.warn("index [{}] is in RED state", health.getIndex());
-                            } else {
-                                logger.debug("index [{}] is in [{}] state", health.getIndex(), health.getStatus().name());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-*/
-
         initTemplates();
         initMappings();
         initAliases();
@@ -613,7 +578,27 @@ private RestClient buildRestClient() throws Exception {
             hosts.add(new HttpHost(addressPort.v1(), addressPort.v2(), "http"));
         }
 
-        return RestClient.builder(hosts.toArray(new HttpHost[]{})).build();
+        RestClientBuilder rcb = RestClient.builder(hosts.toArray(new HttpHost[]{}));
+
+        // We need to check if we have a user security property
+        String securedUser = properties != null ? properties.getProperty(XPACK_USER, null) : null;
+        if (securedUser != null) {
+            // We split the username and the password
+            String[] split = securedUser.split(":");
+            if (split.length < 2) {
+                throw new IllegalArgumentException(XPACK_USER + " must have the form username:password");
+            }
+            String username = split[0];
+            String password = split[1];
+
+            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY,
+                    new UsernamePasswordCredentials(username, password));
+
+            rcb.setHttpClientConfigCallback(hcb -> hcb.setDefaultCredentialsProvider(credentialsProvider));
+        }
+
+        return rcb.build();
 	}
 
 	/**

src/main/java/fr/pilato/spring/elasticsearch/ElasticsearchTransportClientFactoryBean.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.aop.framework.ProxyFactory;
@@ -646,7 +647,18 @@ private Client buildClient() throws Exception {
 			logger.debug("Adding plugin [{}]", plugin);
             pluginClasses.add((Class<? extends Plugin>) ClassUtils.resolveClassName(plugin, Thread.currentThread().getContextClassLoader()));
 		}
-        PreBuiltTransportClient client = new PreBuiltTransportClient(settingsBuilder.build(), pluginClasses);
+
+        TransportClient client;
+
+        // We need to check if we have a user security property
+        String securedUser = properties != null ? properties.getProperty(XPACK_USER, null) : null;
+        if (securedUser != null) {
+            logger.debug("Building a Secured XPack Transport Client");
+            client = new PreBuiltXPackTransportClient(settingsBuilder.build(), pluginClasses);
+        } else {
+            logger.debug("Building a Transport Client");
+            client = new PreBuiltTransportClient(settingsBuilder.build(), pluginClasses);
+        }
 
         for (String esNode : esNodes) {
             client.addTransportAddress(toAddress(esNode));