Skip to content

Commit

Permalink
Merge pull request #355 from mesos/enhancement/334-UseJars
Browse files Browse the repository at this point in the history
Enhancement/334 use jars
  • Loading branch information
Phil Winder committed Oct 19, 2015
2 parents d64f3ac + 9e9f655 commit 0521ca0
Show file tree
Hide file tree
Showing 8 changed files with 285 additions and 41 deletions.
30 changes: 30 additions & 0 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,14 @@ Usage: (Options preceded by an asterisk are required) [options]
The path to the file which contains the secret for the principal
(password). Password in file must not have a newline.
Default: <empty string>
--frameworkUseDocker
The framework will use docker if true, or jar files if false. If false,
the user must ensure that the scheduler jar is on all slaves.
Default: true
--javaHome
(Only when frameworkUseDocker is false) When starting in jar mode, if java
is not on the path, you can specify the path here.
Default: <empty string>
--webUiPort
TCP port for web ui interface.
Default: 31100
Expand Down Expand Up @@ -230,6 +238,28 @@ To use framework Auth, and if you are using docker, you must mount a docker volu
```
Please note that the framework password file must only contain the password (no username) and must not have a newline at the end of the file. (Marathon bugs)

### Using JAR files instead of docker images
It is strongly recommended that you use the containerized version of Mesos Elasticsearch. This ensures that all dependencies are met. Limited support is available for the jar version, since many issues are due to OS configuration. However, if you can't or don't want to use containers, use the raw JAR files in the following way:
0. Requirements: Java 8, Apache Mesos.
1. Read through the developer documentation and build the jars.
2. Copy the `./scheduler/build/libs/elasticsearch-mesos-scheduler-$VERSION.jar to all slaves in cluster. (The executor jar is inside and hosted by the scheduler jar)
3. Set the CLI parameter frameworkUseDocker to false. Set the javaHome CLI parameter if necessary.
4. Run the jar file manually, or use marathon. Normal command line parameters apply. For example:
```
{
"id": "elasticsearch-jar",
"cpus": 0.5,
"mem": 512,
"instances": 1,
"cmd": "/opt/mesosphere/bin/java -jar /home/core/elasticsearch-mesos-scheduler-0.4.3.jar --javaHome /opt/mesosphere/bin/java --frameworkName esjar --frameworkUseDocker false --zookeeperMesosUrl zk://1.2.3.4:2181",
"env": {
"JAVA_OPTS": "-Xms128m -Xmx256m"
},
"ports": [31100],
"requirePorts": true
}
```

### User Interface

The web based user interface is available on port 31100 of the scheduler by default. It displays real time information about the tasks running in the cluster and a basic configuration overview of the cluster.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import org.apache.mesos.elasticsearch.common.zookeeper.formatter.ZKFormatter;
import org.apache.mesos.elasticsearch.common.zookeeper.parser.ZKAddressParser;

import java.net.InetSocketAddress;

/**
* Holder object for framework configuration.
*/
Expand Down Expand Up @@ -40,6 +42,8 @@ public class Configuration {
public static final String FRAMEWORK_PRINCIPAL = "--frameworkPrincipal";
public static final String FRAMEWORK_SECRET_PATH = "--frameworkSecretPath";
private static final Logger LOGGER = Logger.getLogger(Configuration.class);
public static final String FRAMEWORK_USE_DOCKER = "--frameworkUseDocker";
public static final String JAVA_HOME = "--javaHome";
@Parameter(names = {EXECUTOR_HEALTH_DELAY}, description = "The delay between executor healthcheck requests (ms).", validateValueWith = CLIValidators.PositiveLong.class)
private static Long executorHealthDelay = 30000L;
// **** ZOOKEEPER
Expand Down Expand Up @@ -77,6 +81,11 @@ public class Configuration {
private String frameworkPrincipal = "";
@Parameter(names = {FRAMEWORK_SECRET_PATH}, description = "The path to the file which contains the secret for the principal (password). Password in file must not have a newline.")
private String frameworkSecretPath = "";
@Parameter(names = {FRAMEWORK_USE_DOCKER}, arity = 1, description = "The framework will use docker if true, or jar files if false. If false, the user must ensure that the scheduler jar is on all slaves.")
private Boolean frameworkUseDocker = true;
private InetSocketAddress frameworkFileServerAddress;
@Parameter(names = {JAVA_HOME}, description = "(Only when frameworkUseDocker is false) When starting in jar mode, if java is not on the path, you can specify the path here.", validateWith = CLIValidators.NotEmptyString.class)
private String javaHome = "";
// ****************** Runtime configuration **********************

public Configuration(String... args) {
Expand Down Expand Up @@ -210,6 +219,26 @@ public String getFrameworkPrincipal() {
return frameworkPrincipal;
}

public Boolean frameworkUseDocker() {
return frameworkUseDocker;
}

public String getFrameworkFileServerAddress() {
return "http://" + frameworkFileServerAddress.getHostName() + ":" + frameworkFileServerAddress.getPort();
}

public void setFrameworkFileServerAddress(InetSocketAddress addr) {
if (addr != null) {
frameworkFileServerAddress = addr;
} else {
LOGGER.error("Could not set webserver address. Was null.");
}
}

public String getJavaHome() {
return javaHome.replaceAll("java$", "").replaceAll("/$", "") + "/";
}

/**
* Ensures that the number is > than the EXECUTOR_HEALTH_DELAY
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.apache.mesos.state.ZooKeeperState;
import org.springframework.boot.builder.SpringApplicationBuilder;

import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.concurrent.TimeUnit;

Expand All @@ -31,6 +32,16 @@ public void run(String[] args) {

configuration = new Configuration(args);

if (!configuration.frameworkUseDocker()) {
try {
final SimpleFileServer simpleFileServer = new SimpleFileServer();
simpleFileServer.run();
configuration.setFrameworkFileServerAddress(simpleFileServer.getAddress());
} catch (UnknownHostException e) {
throw new IllegalStateException("Unable to start file server. See stack trace.", e);
}
}

final SerializableZookeeperState zookeeperStateDriver = new SerializableZookeeperState(new ZooKeeperState(
configuration.getMesosStateZKURL(),
configuration.getZookeeperCLI().getZookeeperMesosTimeout(),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package org.apache.mesos.elasticsearch.scheduler;

import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;

/**
* Simple file server for distributing jars and zips across the cluster
*/
public class SimpleFileServer implements Runnable {
private static final Logger LOGGER = Logger.getLogger(SimpleFileServer.class);
public static final String ES_EXECUTOR_JAR = "elasticsearch-mesos-executor.jar";
private HttpServer server;

private static void writeClassPathResource(HttpExchange t, String classPathResource) throws IOException {
InputStream in = SimpleFileServer.class.getClassLoader().getResourceAsStream(classPathResource);

// Must send headers before body.
t.sendResponseHeaders(200, 0);
OutputStream os = t.getResponseBody();
IOUtils.copy(in, os);
os.flush();

os.close();
in.close();
}

public void serve() throws IOException {
server = HttpServer.create(new InetSocketAddress(0), 0); // Pick a random available port
server.createContext("/info", new InfoHandler());
server.createContext("/get", new GetHandler());
server.setExecutor(null); // creates a default executor
server.start();
}

public InetSocketAddress getAddress() throws UnknownHostException {
if (server != null) {
return new InetSocketAddress(InetAddress.getLocalHost().getHostName(), server.getAddress().getPort());
} else {
return null;
}
}

@Override
public void run() {
try {
this.serve();
LOGGER.info("Running Executor JAR file server on: " + this.getAddress().getHostName() + ":" + this.getAddress().getPort());
} catch (IOException e) {
LOGGER.error("Elasticsearch file server stopped", e);
e.printStackTrace();
}
}

static class InfoHandler implements HttpHandler {
public void handle(HttpExchange t) throws IOException {
String response = "Use /get to download the executor jar";
t.sendResponseHeaders(200, response.length());

OutputStream os = t.getResponseBody();
IOUtils.write(response, os);
os.close();
}
}

static class GetHandler implements HttpHandler {
public void handle(HttpExchange t) throws IOException {

Headers h = t.getResponseHeaders();
h.add("Content-Type", "application/octet-stream");

writeClassPathResource(t, ES_EXECUTOR_JAR);
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -84,17 +84,20 @@ private ByteString toData(String hostname, String ipAddress, ZonedDateTime zoned
}

private Protos.ExecutorInfo.Builder newExecutorInfo(Configuration configuration) {
return Protos.ExecutorInfo.newBuilder()
Protos.ExecutorInfo.Builder executorInfoBuilder = Protos.ExecutorInfo.newBuilder()
.setExecutorId(Protos.ExecutorID.newBuilder().setValue(UUID.randomUUID().toString()))
.setFrameworkId(frameworkState.getFrameworkID())
.setName("elasticsearch-executor-" + UUID.randomUUID().toString())
.setCommand(newCommandInfo(configuration))
.setContainer(Protos.ContainerInfo.newBuilder()
.setType(Protos.ContainerInfo.Type.DOCKER)
.setDocker(Protos.ContainerInfo.DockerInfo.newBuilder().setImage(configuration.getExecutorImage()).setForcePullImage(configuration.getExecutorForcePullImage()))
.addVolumes(Protos.Volume.newBuilder().setHostPath(SETTINGS_PATH_VOLUME).setContainerPath(SETTINGS_PATH_VOLUME).setMode(Protos.Volume.Mode.RO)) // Temporary fix until we get a data container.
.addVolumes(Protos.Volume.newBuilder().setContainerPath(SETTINGS_DATA_VOLUME_CONTAINER).setHostPath(configuration.getDataDir()).setMode(Protos.Volume.Mode.RW).build())
.build());
.setCommand(newCommandInfo(configuration));
if (configuration.frameworkUseDocker()) {
executorInfoBuilder.setContainer(Protos.ContainerInfo.newBuilder()
.setType(Protos.ContainerInfo.Type.DOCKER)
.setDocker(Protos.ContainerInfo.DockerInfo.newBuilder().setImage(configuration.getExecutorImage()).setForcePullImage(configuration.getExecutorForcePullImage()))
.addVolumes(Protos.Volume.newBuilder().setHostPath(SETTINGS_PATH_VOLUME).setContainerPath(SETTINGS_PATH_VOLUME).setMode(Protos.Volume.Mode.RO)) // Temporary fix until we get a data container.
.addVolumes(Protos.Volume.newBuilder().setContainerPath(SETTINGS_DATA_VOLUME_CONTAINER).setHostPath(configuration.getDataDir()).setMode(Protos.Volume.Mode.RW).build())
.build());
}
return executorInfoBuilder;
}

private Protos.CommandInfo.Builder newCommandInfo(Configuration configuration) {
Expand All @@ -108,11 +111,29 @@ private Protos.CommandInfo.Builder newCommandInfo(Configuration configuration) {
addIfNotEmpty(args, ElasticsearchCLIParameter.ELASTICSEARCH_SETTINGS_LOCATION, configuration.getElasticsearchSettingsLocation());
addIfNotEmpty(args, ElasticsearchCLIParameter.ELASTICSEARCH_CLUSTER_NAME, configuration.getElasticsearchClusterName());
args.addAll(asList(ElasticsearchCLIParameter.ELASTICSEARCH_NODES, Integer.toString(configuration.getElasticsearchNodes())));
return Protos.CommandInfo.newBuilder()
.setShell(false)
.addAllArguments(args)
.setEnvironment(Protos.Environment.newBuilder().addAllVariables(executorEnvironmentalVariables.getList()))
.setContainer(Protos.CommandInfo.ContainerInfo.newBuilder().setImage(configuration.getExecutorImage()).build());

Protos.CommandInfo.Builder commandInfoBuilder = Protos.CommandInfo.newBuilder()
.setEnvironment(Protos.Environment.newBuilder().addAllVariables(executorEnvironmentalVariables.getList()));

if (configuration.frameworkUseDocker()) {
commandInfoBuilder
.setShell(false)
.addAllArguments(args)
.setContainer(Protos.CommandInfo.ContainerInfo.newBuilder().setImage(configuration.getExecutorImage()).build());
} else {
String address = configuration.getFrameworkFileServerAddress();
if (address == null) {
throw new NullPointerException("Webserver address is null");
}
String httpPath = address + "/get/" + SimpleFileServer.ES_EXECUTOR_JAR;
LOGGER.debug("Using file server: " + httpPath);
commandInfoBuilder
.setValue(configuration.getJavaHome() + "java $JAVA_OPTS -jar ./" + SimpleFileServer.ES_EXECUTOR_JAR)
.addAllArguments(args)
.addUris(Protos.CommandInfo.URI.newBuilder().setValue(httpPath));
}

return commandInfoBuilder;
}

private void addIfNotEmpty(List<String> args, String key, String value) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ public List<Protos.Environment.Variable> getList() {
* @param configuration
*/
private void populateEnvMap(Configuration configuration) {
addToList(native_mesos_library_key, native_mesos_library_path);
if (configuration.frameworkUseDocker()) {
addToList(native_mesos_library_key, native_mesos_library_path);
}
addToList(JAVA_OPTS, getHeapSpaceString(configuration));
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package org.apache.mesos.elasticsearch.scheduler;

import org.apache.mesos.elasticsearch.common.cli.ZookeeperCLIParameter;
import org.junit.Test;

import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;

/**
* Tests
**/
public class ConfigurationTest {
@Test
public void shouldReturnValidServerPath() throws UnknownHostException {
Configuration configuration = new Configuration(ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, "aa");
String localhost = "localhost";
int port = 1234;
configuration.setFrameworkFileServerAddress(new InetSocketAddress(localhost, port));
assertEquals("http://" + localhost + ":" + port, configuration.getFrameworkFileServerAddress());
}

@Test
public void shouldNotHaveDefaultInetAddressToStringMethod() throws UnknownHostException {
Configuration configuration = new Configuration(ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, "aa");
int port = 1234;
configuration.setFrameworkFileServerAddress(new InetSocketAddress(InetAddress.getLocalHost().getHostName(), port));
assertFalse(configuration.getFrameworkFileServerAddress().replace("http://", "").contains("/"));
}

@Test
public void shouldProvideJavaHomeWithEndSlashAndWithoutJava() {
Configuration configuration = new Configuration(ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, "aa", Configuration.JAVA_HOME, "/usr/bin/java");
assertEquals("/usr/bin/", configuration.getJavaHome());
configuration = new Configuration(ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, "aa", Configuration.JAVA_HOME, "/usr/bin/");
assertEquals("/usr/bin/", configuration.getJavaHome());
configuration = new Configuration(ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, "aa", Configuration.JAVA_HOME, "/usr/bin");
assertEquals("/usr/bin/", configuration.getJavaHome());
}
}

0 comments on commit 0521ca0

Please sign in to comment.