working transport module with slightly improved documentation

foundit · Sep 6, 2013 · f02a2ad · f02a2ad
1 parent 8f77ca3
commit f02a2ad
Show file tree

Hide file tree

Showing 14 changed files with 359 additions and 50 deletions.
diff --git a/Distributing.md b/Distributing.md
@@ -0,0 +1,9 @@
+We usually use the "developers@found.no" pgp-key when publishing. sbt-pgp seems to
+break if the GPG-keyring contains more than one key, or we try to sign with something
+else than the default. If that had not been the case, the following should have worked
+regardless of the number of keys stored in the local keyring:
+
+    set usePgpKeyHex("440065AC58944314")
+
+However, it currently does not. Until this suddenly starts working, make sure that
+there is only one key in the keyring, or that the required key is the default chosen.
diff --git a/Readme.md b/Readme.md
@@ -0,0 +1,108 @@
+# Found Elasticsearch Transport Module
+
+A transport module that works with Found Elasticsearch.
+
+## Installing
+
+To install, add a dependency to this module in your build system :
+
+```xml
+    <dependency>
+        <groupId>no.found.elasticsearch</groupId>
+        <artifactId>elasticsearch-transport-module</artifactId>
+        <version>1.1.18-SNAPSHOT</version>
+    </dependency>
+```
+
+The module is enabled by adding this project as a dependency to your application
+and setting the ``transport.type`` setting in Elasticsearch to
+``no.found.elasticsearch.transport.netty.FoundNettyTransportModule``.
+
+Note: This is not a standard Elasticsearch plugin, it just needs to be on the
+application classpath.
+
+## Configuring an API key.
+
+In order to use this module, you must configure one or more API-keys. API-keys
+are stored as a list of acceptable keys under ``api_keys`` at the root level
+of the ACL. For example:
+
+```yaml
+default: deny
+
+api_keys:
+    - s6aW9aAMZjDMbuhj
+    - 6hKZsTqBru9KnVaW
+
+auth:
+  users:
+    ...
+
+rules:
+  - ...
+```
+
+In the above example, both ``s6aW9aAMZjDMbuhj`` and ``6hKZsTqBru9KnVaW`` would be
+valid API-keys.
+
+## New Elasticsearch settings
+
+New settings introduced by this module:
+
+* **transport.found.host-suffixes**: A comma-separated list of host suffixes that
+ trigger our attempt to authenticate with Found Elasticsearch. Defaults to
+ {@code foundcluster.com,found.no}".
+
+* **transport.found.ssl-ports**: A comma-separated list of ports that trigger our
+ SSL support. Defaults to {@code 9343}".
+
+* **transport.found.api-key**: An API-key which is used to authorize this client
+ when connecting to Found Elasticsearch. API-keys are managed via the console as
+ a list of Strings under the root level key "api_keys". Defaults to
+ {@code missing-api-key}
+
+* **transport.found.ssl.unsafe_allow_self_signed**: Whether to accept self-signed
+ certificates when using SSL. This is unsafe and allows for MITM-attacks, but
+ may be useful for testing. Defaults to {@code false}.
+
+**The transport is backwards-compatible with the default transport.**
+
+## Example configuration
+
+```java
+// Build the settings for our transport client.
+Settings settings = ImmutableSettings.settingsBuilder()
+    // Setting "transport.type" enables this module:
+    .put("transport.type", "no.found.elasticsearch.transport.netty.FoundNettyTransportModule")
+    // Create an api key via the console and add it here:
+    .put("transport.found.api-key", "YOUR_API_KEY")
+
+    // Used by Elasticsearch:
+    .put("cluster.name", "YOUR_CLUSTER_ID")
+    .put("client.transport.ignore_cluster_name", false)
+
+    .build();
+
+// Instantiate a TransportClient and add Found Elasticsearch to the list of addresses to connect to.
+// Only port 9343 (SSL-encrypted) is currently supported.
+Client client = new TransportClient(settings)
+    .addTransportAddress(new InetSocketTransportAddress("YOUR_CLUSTER_ID-REGION.foundcluster.com", 9343));
+```
+
+## Example usage:
+
+```java
+while(true) {
+    try {
+        System.out.print("Getting cluster health... "); System.out.flush();
+        ActionFuture<ClusterHealthResponse> healthFuture = client.admin().cluster().health(Requests.clusterHealthRequest());
+        ClusterHealthResponse healthResponse = healthFuture.get(5, TimeUnit.SECONDS);
+        System.out.println("Got response: " + healthResponse.getStatus());
+    } catch(Throwable t) {
+        System.out.println("Error: " + t);
+    }
+    try {
+        Thread.sleep(3000);
+    } catch (InterruptedException ie) { ie.printStackTrace(); }
+}
+```
diff --git a/build.sbt b/build.sbt
diff --git a/...in/scala/no/found/esproxy/Bootstrap.scala → ...d/elasticsearch/transport/Bootstrap.scala b/...in/scala/no/found/esproxy/Bootstrap.scala → ...d/elasticsearch/transport/Bootstrap.scala
@@ -1,35 +1,32 @@
-package no.found.esproxy
-
+package no.found.elasticsearch.transport
 
+import java.util.concurrent.TimeUnit
+import java.lang.Throwable
 import org.elasticsearch.client.transport.TransportClient
-import org.elasticsearch.common.transport.InetSocketTransportAddress
 import org.elasticsearch.common.settings.ImmutableSettings
+import org.elasticsearch.common.transport.InetSocketTransportAddress
 import org.elasticsearch.client.Requests
-import java.util.concurrent.TimeUnit
 
 
 object Bootstrap extends App {
   val settings = ImmutableSettings.settingsBuilder()
-    .put("cluster.name", "c70ce4781ef74c0aa41e315241556a5b")
-    .put("client.transport.ignore_cluster_name", true)
-
-    .put("client.transport.sniff", false)
+    //.put("cluster.name", "20ef6fed67864b4aba920f3a2c45d7f3")
+    //.put("client.transport.ignore_cluster_name", false)
 
     .put("transport.type", "no.found.elasticsearch.transport.netty.FoundNettyTransportModule")
     .put("transport.found.host-suffixes", ".localhacks.com,.foundcluster.com")
-    .put("transport.found.ssl.unsafe_allow_self_signed", true)
-    .put("transport.found.ssl-ports", "9343")
 
     .put("transport.found.api-key", "foobarbaz")
 
-    .put("transport.netty.connections_per_node.low", 1)
-    .put("transport.netty.connections_per_node.med", 1)
-    .put("transport.netty.connections_per_node.high", 1)
+    //.put("transport.netty.connections_per_node.low", 1)
+    //.put("transport.netty.connections_per_node.med", 1)
+    //.put("transport.netty.connections_per_node.high", 1)
 
     .build()
 
   var client = new TransportClient(settings)
-    .addTransportAddress(new InetSocketTransportAddress("c70ce4781ef74c0aa41e315241556a5b.localhacks.com", 9343))
+    .addTransportAddress(new InetSocketTransportAddress("20ef6fed67864b4aba920f3a2c45d7f3-local-1.localhack2s.com", 9300))
+    //.addTransportAddress(new InetSocketTransportAddress("92e121b43d8eb1e2be00992154ecfa3c-eu-west-1.foundcluster.com", 9343))
 
   while(true) {
     try {

diff --git a/project/Build.scala b/project/Build.scala
@@ -0,0 +1,84 @@
+import sbt.Keys._
+import sbt._
+
+object Build extends Build {
+  val foundOrganizationName = "Found AS"
+  val foundOrgnizationPrefix = "no.found"
+
+  val elasticsearchVersion = "0.90.3"
+
+  val transportOrganization = foundOrgnizationPrefix + ".elasticsearch"
+  val transportName = "elasticsearch-transport-module"
+  val transportVersion = "1.1.18-SNAPSHOT"
+
+  var transportDependencies = Seq[ModuleID]()
+
+  transportDependencies ++= Seq(
+    "org.elasticsearch" % "elasticsearch" % elasticsearchVersion % "provided",
+
+    "junit" % "junit" % "4.11" % "test",
+
+    "org.mockito" % "mockito-all" % "1.9.5" % "test"
+  )
+
+  lazy val root = Project(id = transportName, base=file("."), settings = Project.defaultSettings).settings(
+    organizationName := foundOrganizationName,
+    organization := transportOrganization,
+
+    // don't build with _{scalaVersion}-suffix
+    crossPaths := false,
+
+    publishTo := {
+      val nexus = "https://oss.sonatype.org/"
+      if (version.value.trim.endsWith("SNAPSHOT"))
+        Some("snapshots" at nexus + "content/repositories/snapshots")
+      else
+        Some("releases"  at nexus + "service/local/staging/deploy/maven2")
+    },
+    publishMavenStyle := true,
+    pomIncludeRepository := { _ => false },
+    publishArtifact in Test := false,
+
+    licenses := Seq("MIT" -> url("http://opensource.org/licenses/MIT")),
+
+    homepage := Some(url("https://github.com/foundit/elasticsearch-transport-module")),
+
+    pomExtra := <scm>
+        <url>git@github.com:foundit/elasticsearch-transport-module.git</url>
+        <connection>scm:git:git@github.com:foundit/elasticsearch-transport-module.git</connection>
+      </scm>
+      <developers>
+        <developer>
+          <id>nkvoll</id>
+          <name>Njal Karevoll</name>
+          <url>http://www.found.no</url>
+        </developer>
+      </developers>,
+
+    version := transportVersion,
+
+    scalacOptions in ThisBuild ++= Seq("-unchecked", "-deprecation", "-feature"),
+
+    libraryDependencies := transportDependencies
+  ).settings(net.virtualvoid.sbt.graph.Plugin.graphSettings: _*)
+
+  lazy val integration = Project("integration", file("./integration"))
+    .dependsOn(root % "test->test")
+    .settings(Project.defaultSettings : _*)
+    .settings(
+      organizationName := foundOrganizationName,
+
+      libraryDependencies ++= Seq(
+        "org.elasticsearch" % "elasticsearch" % elasticsearchVersion
+      ),
+
+      // Integration tests are not intended to be run in parallel.
+      parallelExecution in Test := false,
+      logBuffered in Test := false
+    )
+
+  // configure prompt to show current project
+  override lazy val settings = super.settings :+ {
+    shellPrompt := { s => Project.extract(s).currentProject.id + " > " }
+  }
+}
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=0.12.2
+sbt.version=0.13.0
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1 +1,5 @@
-addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.4.0")
+addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.5.1")
+
+addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.7.4")
+
+addSbtPlugin("com.typesafe.sbt" % "sbt-pgp" % "0.8")
diff --git a/src/main/java/no/found/elasticsearch/transport/netty/FoundNettyTransportModule.java b/src/main/java/no/found/elasticsearch/transport/netty/FoundNettyTransportModule.java
@@ -1,16 +1,92 @@
 package no.found.elasticsearch.transport.netty;
 
+import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.netty.FoundNettyTransport;
 
+/**
+ * A transport module that works with Found Elasticsearch.
+ *
+ * <p>Binds the {@link Transport} to {@link FoundNettyTransport}, which replaces
+ * the default client transport with one that does an initial authentication-step
+ * and supports SSL without any external dependencies.</p>
+ *
+ * <p>New settings introduced by this module:</p>
+ *
+ * <ul>
+ *  <li>{@code transport.found.host-suffixes}: A comma-separated list of host suffixes that
+ *  trigger our attempt to authenticate with Found Elasticsearch. Defaults to
+ *  {@code foundcluster.com,found.no}".</li>
+ *
+ *  <li>{@code transport.found.ssl-ports}: A comma-separated list of ports that trigger our
+ *  SSL support. Defaults to {@code 9343}".</li>
+ *
+ *  <li>{@code transport.found.api-key}: An API-key which is used to authorize this client
+ *  when connecting to Found Elasticsearch. API-keys are managed via the console as
+ *  a list of Strings under the root level key "api_keys". Defaults to
+ *  {@code missing-api-key}</li>
+ *
+ *  <li>{@code transport.found.ssl.unsafe_allow_self_signed}: Whether to accept self-signed
+ *  certificates when using SSL. This is unsafe and allows for MITM-attacks, but
+ *  may be useful for testing. Defaults to {@code false}.</li>
+ * </ul>
+ *
+ * <p><b>The transport is backwards-compatible with the default transport.</b></p>
+ *
+ * <p>Example configuration:</p>
+ *
+ * <pre>
+ * {@code // Build the settings for our client.
+ *   Settings settings = ImmutableSettings.settingsBuilder()
+ *       // Setting "transport.type" enables this module:
+ *       .put("transport.type", "no.found.elasticsearch.transport.netty.FoundNettyTransportModule")
+ *       // Create an api key via the console and add it here:
+ *       .put("transport.found.api-key", "YOUR_API_KEY")
+ *
+ *       // Used by Elasticsearch:
+ *       .put("cluster.name", "YOUR_CLUSTER_ID")
+ *       .put("client.transport.ignore_cluster_name", false)
+ *
+ *       .build();
+ *
+ *   // Instantiate a TransportClient and add Found Elasticsearch to the list of addresses to connect to.
+ *   // Only port 9343 (SSL-encrypted) is currently supported.
+ *   Client client = new TransportClient(settings)
+ *       .addTransportAddress(new InetSocketTransportAddress("YOUR_CLUSTER_ID-REGION.foundcluster.com", 9343));
+ * }
+ * </pre>
+ *
+ * <p>Example usage:</p>
+ *
+ * <pre>
+ * {@code while(true) {
+ *      try {
+ *          System.out.print("Getting cluster health... "); System.out.flush();
+ *          ActionFuture<ClusterHealthResponse> healthFuture = client.admin().cluster().health(Requests.clusterHealthRequest());
+ *          ClusterHealthResponse healthResponse = healthFuture.get(5, TimeUnit.SECONDS);
+ *          System.out.println("Got response: " + healthResponse.getStatus());
+ *      } catch(Throwable t) {
+ *          System.out.println("Error: " + t);
+ *      }
+ *      try {
+ *          Thread.sleep(3000);
+ *      } catch (InterruptedException ie) { ie.printStackTrace(); }
+ *  }
+ * }
+ * </pre>
+ */
 public class FoundNettyTransportModule extends AbstractModule {
 
     private final Settings settings;
 
     public FoundNettyTransportModule(Settings settings) {
         this.settings = settings;
+
+        if(settings.getAsBoolean("client.transport.sniff", false)) {
+            throw new ElasticSearchException("The transport client setting \"client.transport.sniff\" is [true], which is not supported by this transport.");
+        }
     }
 
     @Override

diff --git a/src/main/java/no/found/elasticsearch/transport/netty/FoundSSLUtils.java b/src/main/java/no/found/elasticsearch/transport/netty/FoundSSLUtils.java
@@ -1,14 +1,15 @@
 package no.found.elasticsearch.transport.netty;
 
-import org.elasticsearch.common.settings.Settings;
-
 import javax.net.ssl.*;
 import java.net.InetSocketAddress;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 
+/**
+ * Collection of SSL-related utils.
+ */
 public class FoundSSLUtils {
     public static FoundSSLHandler getSSLHandler(boolean unsafeAllowSelfSigned, InetSocketAddress inetSocketAddress) throws NoSuchAlgorithmException {
         String hostString = inetSocketAddress.getHostString();