Permalink
Browse files

Add support for docker containers to aurora

This change adds support for launching docker containers through
aurora.  These changes are based off of the discussion in
https://issues.apache.org/jira/browse/AURORA-633

As of now, a special thermos_executor.sh script is needed to launch
the executor inside docker containers.  A sample aurora file is in
examples/jobs/docker.

In addition, mesos-slave must be run with
`--containerizers=docker,mesos`, the example upstart config in
examples/vagrant/upstart has been updated to reflect this.

More information is in subsequent review request comments.

Bugs closed: AURORA-633

Reviewed at https://reviews.apache.org/r/28920/
  • Loading branch information...
1 parent c1fdbe4 commit 7ba6226faaee53e86e86f010f2aa674a7beb1192 @steveniemitz steveniemitz committed with kevints Jan 24, 2015
Showing with 669 additions and 123 deletions.
  1. +3 −1 Vagrantfile
  2. +18 −1 api/src/main/thrift/org/apache/aurora/gen/api.thrift
  3. +0 −1 config/legacy_untested_classes.txt
  4. +21 −0 docs/configuration-reference.md
  5. +18 −0 docs/deploying-aurora-scheduler.md
  6. +21 −0 examples/jobs/docker/hello_docker.aurora
  7. +0 −5 examples/vagrant/aurorabuild.sh
  8. +3 −0 examples/vagrant/provision-dev-cluster.sh
  9. +4 −2 examples/vagrant/upstart/aurora-scheduler.conf
  10. +2 −1 examples/vagrant/upstart/mesos-slave.conf
  11. +23 −3 src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
  12. +2 −1 src/main/java/org/apache/aurora/scheduler/async/GcExecutorLauncher.java
  13. +59 −8 src/main/java/org/apache/aurora/scheduler/base/CommandUtil.java
  14. +34 −0 src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
  15. +112 −20 src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
  16. +9 −0 src/main/python/apache/aurora/config/schema/base.py
  17. +21 −4 src/main/python/apache/aurora/config/thrift.py
  18. +3 −13 src/main/python/apache/aurora/executor/aurora_executor.py
  19. +54 −11 src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
  20. +55 −14 src/main/python/apache/aurora/executor/common/sandbox.py
  21. +20 −4 src/main/python/apache/aurora/executor/thermos_task_runner.py
  22. +1 −0 src/main/python/apache/thermos/core/runner.py
  23. +6 −1 src/main/resources/scheduler/assets/configSummary.html
  24. +8 −1 src/main/resources/scheduler/assets/js/services.js
  25. +6 −1 src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java
  26. +28 −3 src/test/java/org/apache/aurora/scheduler/base/CommandUtilTest.java
  27. +21 −0 src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
  28. +92 −17 src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java
  29. +3 −0 src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java
  30. +7 −2 src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
  31. +6 −0 src/test/python/apache/aurora/client/cli/test_status.py
  32. +9 −9 src/test/python/apache/aurora/executor/test_thermos_executor.py
View
@@ -22,12 +22,14 @@ VAGRANTFILE_API_VERSION = "2"
Vagrant.require_version ">= 1.5.0"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+ config.vm.hostname = "aurora.local"
config.vm.box = "ubuntu/trusty64"
config.vm.define "devcluster" do |dev|
dev.vm.network :private_network, ip: "192.168.33.7"
dev.vm.provider :virtualbox do |vb|
- vb.customize ["modifyvm", :id, "--memory", "1024"]
+ vb.customize ["modifyvm", :id, "--memory", "2048"]
+ vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
end
dev.vm.provision "shell", path: "examples/vagrant/provision-dev-cluster.sh"
end
@@ -186,6 +186,22 @@ struct ExecutorConfig {
2: string data
}
+/** Describes a mesos container, this is the default */
+struct MesosContainer {
+}
+
+/** Describes a docker container */
+struct DockerContainer {
+ /** The container image to be run */
+ 1: string image
+}
+
+/** Describes a container to be used in a task */
+union Container {
+ 1: MesosContainer mesos
+ 2: DockerContainer docker
+}
+
/** Description of the tasks contained within a job. */
struct TaskConfig {
/** Job task belongs to. */
@@ -211,7 +227,8 @@ struct TaskConfig {
20: set<Constraint> constraints
/** a list of named ports this task requests */
21: set<string> requestedPorts
-
+ /** the container the task should use to execute */
+ 29: optional Container container = { "mesos": {} }
/**
* Custom links to include when displaying this task on the scheduler dashboard. Keys are anchor
* text, values are URLs. Wildcards are supported for dynamic link crafting based on host, ports,
@@ -2,7 +2,6 @@ org/apache/aurora/auth/UnsecureAuthModule$UnsecureCapabilityValidator$1
org/apache/aurora/auth/UnsecureAuthModule$UnsecureCapabilityValidator$2
org/apache/aurora/auth/UnsecureAuthModule$UnsecureSessionValidator
org/apache/aurora/auth/UnsecureAuthModule$UnsecureSessionValidator$1
-org/apache/aurora/Protobufs$1
org/apache/aurora/scheduler/app/SchedulerMain$2
org/apache/aurora/scheduler/app/SchedulerMain$3
org/apache/aurora/scheduler/async/GcExecutorLauncher$1
@@ -29,6 +29,7 @@ Aurora + Thermos Configuration Reference
- [UpdateConfig Objects](#updateconfig-objects)
- [HealthCheckConfig Objects](#healthcheckconfig-objects)
- [Announcer Objects](#announcer-objects)
+ - [Container Objects](#container)
- [Specifying Scheduling Constraints](#specifying-scheduling-constraints)
- [Template Namespaces](#template-namespaces)
- [mesos Namespace](#mesos-namespace)
@@ -299,6 +300,7 @@ resources are allocated.
```ram``` | Integer | Bytes of RAM required by the task.
```disk``` | Integer | Bytes of disk required by the task.
+
Job Schema
==========
@@ -322,6 +324,7 @@ Job Schema
```priority``` | Integer | Preemption priority to give the task (Default 0). Tasks with higher priorities may preempt tasks at lower priorities.
```production``` | Boolean | Whether or not this is a production task backed by quota (Default: False). Production jobs may preempt any non-production job, and may only be preempted by production jobs in the same role and of higher priority. To run jobs at this level, the job role must have the appropriate quota.
```health_check_config``` | ```heath_check_config``` object | Parameters for controlling a task's health checks via HTTP. Only used if a health port was assigned with a command line wildcard.
+ ```container``` | ```Container``` object | An optional container to run all processes inside of.
### Services
@@ -392,6 +395,24 @@ tasks with the same static port allocations from being co-scheduled.
External constraints such as slave attributes should be used to enforce such
guarantees should they be needed.
+### Container Object
+
+*Note: The only container type currently supported is "docker". Docker support is currently EXPERIMENTAL.*
+*Note: In order to correctly execute processes inside a job, the Docker container must have python 2.7 installed.*
+
+Describes the container the job's processes will run inside.
+
+ param | type | description
+ ----- | :----: | -----------
+ ```docker``` | Docker | A docker container to use.
+
+### Docker Object
+
+ param | type | description
+ ----- | :----: | -----------
+ ```image``` | String | The name of the docker image to execute. If the image does not exist locally it will be pulled with ```docker pull```.
+
+
Specifying Scheduling Constraints
=================================
@@ -12,6 +12,7 @@ machines. This guide helps you get the scheduler set up and troubleshoot some c
- [Initializing the Replicated Log](#initializing-the-replicated-log)
- [Storage Performance Considerations](#storage-performance-considerations)
- [Network considerations](#network-considerations)
+ - [Considerations for running jobs in docker](#considerations-for-running-jobs-in-docker)
- [Running Aurora](#running-aurora)
- [Maintaining an Aurora Installation](#maintaining-an-aurora-installation)
- [Monitoring](#monitoring)
@@ -144,6 +145,23 @@ to ZooKeeper) or explicitly set in the startup script as follows:
export LIBPROCESS_PORT=8083
# ...
+### Considerations for running jobs in docker containers
+*Note: Docker support is currently EXPERIMENTAL.*
+
+In order for Aurora to launch jobs using docker containers, a few extra configuration options
+must be set. The [docker containerizer](http://mesos.apache.org/documentation/latest/docker-containerizer/)
+must be enabled on the mesos slaves by launching them with the `--containerizers=docker,mesos` option.
+
+By default, aurora will configure mesos to copy the file specified in `-thermos_executor_path`
+into the container's sandbox. If using a wrapper script to launch the thermos executor,
+specify the path to the wrapper in that arguement. In addition, the executor pex itself must be
+included in `-thermos_executor_resources` option. Doing so will ensure that both the wrapper script
+and executor are correctly copied into the sandbox. In addition, ensure the wrapper script does not
+access resources outside of the sandbox, because when running inside a docker container they will not
+exist.
+
+In order to correctly execute processes inside a job, the docker container must have python 2.7 installed.
+
## Running Aurora
Configure a supervisor like [Monit](http://mmonit.com/monit/) or
[supervisord](http://supervisord.org/) to run the created `scheduler.sh` file and restart it
@@ -0,0 +1,21 @@
+hello_world_proc = Process(
+ name="hello_process",
+ cmdline="""
+while true; do
+ echo -n "Hello world! The time is now: " && date
+ sleep 10
+done
+""")
+
+hello_world_docker = Task(
+ name = 'hello docker',
+ processes = [hello_world_proc],
+ resources = Resources(cpu = 1, ram = 1*MB, disk=8*MB))
+
+jobs = [
+ Service(cluster = 'devcluster',
+ environment = 'devel',
+ role = 'docker-test',
+ name = 'hello_docker',
+ task = hello_world_docker,
+ container = Container(docker = Docker(image = 'python:2.7')))]
@@ -71,11 +71,6 @@ with contextlib.closing(zipfile.ZipFile('dist/thermos_executor.pex', 'a')) as zf
zf.write('dist/thermos_runner.pex', 'apache/aurora/executor/resources/thermos_runner.pex')
EOF
- cat <<EOF > $DIST_DIR/thermos_executor.sh
-#!/usr/bin/env bash
-exec /home/vagrant/aurora/dist/thermos_executor.pex --announcer-enable --announcer-ensemble localhost:2181
-EOF
- chmod +x $DIST_DIR/thermos_executor.sh
chmod +x /home/vagrant/aurora/dist/thermos_executor.pex
}
@@ -13,12 +13,15 @@
# limitations under the License.
#
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
+echo deb https://get.docker.com/ubuntu docker main > /etc/apt/sources.list.d/docker.list
apt-get update
apt-get -y install \
curl \
git \
libcurl4-openssl-dev \
libsasl2-dev \
+ lxc-docker \
openjdk-7-jdk \
python-dev \
zookeeper
@@ -35,8 +35,10 @@ exec $DIST_DIR/install/aurora-scheduler/bin/aurora-scheduler \
-native_log_zk_group_path=/aurora/replicated-log \
-native_log_file_path=$AURORA_HOME/scheduler/db \
-backup_dir=$AURORA_HOME/scheduler/backups \
- -thermos_executor_path=$DIST_DIR/thermos_executor.sh \
+ -thermos_executor_path=$DIST_DIR/thermos_executor.pex \
+ -thermos_executor_flags="--announcer-enable --announcer-ensemble localhost:2181" \
-gc_executor_path=$DIST_DIR/gc_executor.pex \
-enable_beta_updater=true \
-vlog=INFO \
- -logtostderr
+ -logtostderr \
+ -allowed_container_types=MESOS,DOCKER
@@ -28,4 +28,5 @@ exec /usr/local/sbin/mesos-slave --master=zk://$ZK_HOST:2181/mesos/master \
--hostname=$MY_HOST \
--attributes="host:$MY_HOST;rack:a" \
--resources="cpus:4;mem:1024;disk:20000" \
- --work_dir="/var/lib/mesos"
+ --work_dir="/var/lib/mesos" \
+ --containerizers=docker,mesos
@@ -83,10 +83,24 @@
@CmdLine(name = "serverset_path", help = "ZooKeeper ServerSet path to register at.")
private static final Arg<String> SERVERSET_PATH = Arg.create();
- @NotNull
- @CmdLine(name = "thermos_executor_path", help = "Path to the thermos executor launch script.")
+ @CmdLine(name = "thermos_executor_path", help = "Path to the thermos executor entry point.")
private static final Arg<String> THERMOS_EXECUTOR_PATH = Arg.create();
+ @CmdLine(name = "thermos_executor_resources",
+ help = "A comma seperated list of additional resources to copy into the sandbox."
+ + "Note: if thermos_executor_path is not the thermos_executor.pex file itself, "
+ + "this must include it.")
+ private static final Arg<List<String>> THERMOS_EXECUTOR_RESOURCES =
+ Arg.<List<String>>create(ImmutableList.<String>of());
+
+ @CmdLine(name = "thermos_executor_flags",
+ help = "Extra arguments to be passed to the thermos executor")
+ private static final Arg<String> THERMOS_EXECUTOR_FLAGS = Arg.create(null);
+
+ @CmdLine(name = "thermos_observer_root",
+ help = "Path to the thermos observer root (by default /var/run/thermos.)")
+ private static final Arg<String> THERMOS_OBSERVER_ROOT = Arg.create("/var/run/thermos");
+
@CmdLine(name = "auth_module",
help = "A Guice module to provide auth bindings. NOTE: The default is unsecure.")
private static final Arg<? extends Class<? extends Module>> AUTH_MODULE =
@@ -199,8 +213,14 @@ protected void configure() {
EXECUTOR_OVERHEAD_RAM.get(),
Amount.of(0L, Data.MB),
0);
+
bind(ExecutorSettings.class)
- .toInstance(new ExecutorSettings(THERMOS_EXECUTOR_PATH.get(), executorOverhead));
+ .toInstance(new ExecutorSettings(
+ THERMOS_EXECUTOR_PATH.get(),
+ THERMOS_EXECUTOR_RESOURCES.get(),
+ THERMOS_OBSERVER_ROOT.get(),
+ Optional.fromNullable(THERMOS_EXECUTOR_FLAGS.get()),
+ executorOverhead));
}
})
.add(getMesosModules())
@@ -27,6 +27,7 @@
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.protobuf.ByteString;
import com.twitter.common.quantity.Amount;
@@ -159,7 +160,7 @@ static TaskInfo makeGcTask(
.setName(EXECUTOR_NAME)
.setSource(sourceName)
.addAllResources(GC_EXECUTOR_TASK_RESOURCES.toResourceList())
- .setCommand(CommandUtil.create(gcExecutorPath));
+ .setCommand(CommandUtil.create(gcExecutorPath, ImmutableList.<String>of()));
byte[] data;
try {
@@ -13,6 +13,12 @@
*/
package org.apache.aurora.scheduler.base;
+import java.util.List;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
import com.twitter.common.base.MorePreconditions;
import org.apache.mesos.Protos.CommandInfo;
@@ -23,11 +29,24 @@
*/
public final class CommandUtil {
+ private static final Function<String, URI> STRING_TO_URI_RESOURCE = new Function<String, URI>() {
+ @Override
+ public URI apply(String resource) {
+ return URI.newBuilder().setValue(resource).setExecutable(true).build();
+ }
+ };
+
private CommandUtil() {
// Utility class.
}
- private static String uriBasename(String uri) {
+ /**
+ * Gets the last part of the path of a URI.
+ *
+ * @param uri URI to parse
+ * @return The last segment of the URI.
+ */
+ public static String uriBasename(String uri) {
int lastSlash = uri.lastIndexOf('/');
if (lastSlash == -1) {
return uri;
@@ -43,15 +62,47 @@ private static String uriBasename(String uri) {
* Creates a description of a command that will fetch and execute the given URI to an executor
* binary.
*
- * @param executorUri URI to the executor.
- * @return A command that will fetch and execute the executor.
+ * @param executorUri A URI to the executor
+ * @param executorResources A list of URIs to be fetched into the sandbox with the executor.
+ * @return A populated CommandInfo with correct resources set and command set.
*/
- public static CommandInfo create(String executorUri) {
+ public static CommandInfo create(String executorUri, List<String> executorResources) {
+ return create(
+ executorUri,
+ executorResources,
+ "./",
+ Optional.<String>absent()).build();
+ }
+
+ /**
+ * Creates a description of a command that will fetch and execute the given URI to an executor
+ * binary.
+ *
+ * @param executorUri A URI to the executor
+ * @param executorResources A list of URIs to be fetched into the sandbox with the executor.
+ * @param commandBasePath The relative base path of the executor.
+ * @param extraArguments Extra command line arguments to add to the generated command.
+ * @return A CommandInfo.Builder populated with resources and a command.
+ */
+ public static CommandInfo.Builder create(
+ String executorUri,
+ List<String> executorResources,
+ String commandBasePath,
+ Optional<String> extraArguments) {
+
+ Preconditions.checkNotNull(executorResources);
MorePreconditions.checkNotBlank(executorUri);
+ MorePreconditions.checkNotBlank(commandBasePath);
+ CommandInfo.Builder builder = CommandInfo.newBuilder();
+
+ builder.addAllUris(Iterables.transform(executorResources, STRING_TO_URI_RESOURCE));
+ builder.addUris(STRING_TO_URI_RESOURCE.apply(executorUri));
- return CommandInfo.newBuilder()
- .addUris(URI.newBuilder().setValue(executorUri).setExecutable(true))
- .setValue("./" + uriBasename(executorUri))
- .build();
+ String cmdLine = commandBasePath
+ + uriBasename(executorUri)
+ + " " + extraArguments.or("");
+ return builder
+ .setValue(cmdLine.trim())
+ .setShell(true);
}
}
Oops, something went wrong.

0 comments on commit 7ba6226

Please sign in to comment.