Added Strict mode support (#580)

* Added strict mode support for hello-world in V0 non-http * Refactored Kafka and added strict mode support * Turned off support for configuring custom kafka zk node * Added strict mode support for HDFS * Adds strict mode support to template framework * Adds user to kafka svc.yml * Added strict mode support for Cassandra * Added strict mode support for elastic
mesosphere · Feb 17, 2017 · d7e61d7 · d7e61d7
1 parent 437a779
commit d7e61d7
Show file tree

Hide file tree

Showing 28 changed files with 231 additions and 101 deletions.
diff --git a/frameworks/cassandra/build.gradle b/frameworks/cassandra/build.gradle
@@ -20,8 +20,8 @@ ext {
 }
 
 dependencies {
-    compile "mesosphere:scheduler:0.11.0"
-    compile "mesosphere:executor:0.11.0"
+    compile project(":scheduler")
+    compile project(":executor")
     compile group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.21'
     testCompile project(":testing")
 }

diff --git a/frameworks/cassandra/src/main/dist/svc.yml b/frameworks/cassandra/src/main/dist/svc.yml
@@ -1,4 +1,7 @@
 name: {{FRAMEWORK_NAME}}
+scheduler:
+  principal: {{FRAMEWORK_PRINCIPAL}}
+  user: {{SERVICE_USER}}
 pods:
   node:
     count: {{NODES}}

diff --git a/frameworks/cassandra/universe/config.json b/frameworks/cassandra/universe/config.json
@@ -10,10 +10,30 @@
           "type": "string",
           "default": "cassandra"
         },
+        "cmd_prefix" : {
+          "description":"A generic prefix to start the scheduler.",
+          "type":"string",
+          "default":"export LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib:$LD_LIBRARY_PATH; export MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so); export JAVA_HOME=$(ls -d $MESOS_SANDBOX/jre*/); export JAVA_HOME=${JAVA_HOME%/}; export PATH=$(ls -d $JAVA_HOME/bin):$PATH"
+        },
+        "secret_name": {
+          "description":"Name of the Secret Store credentials to use for DC/OS service authentication. This should be left empty unless service authentication is needed.",
+          "type":"string",
+          "default":""
+        },
+        "principal": {
+          "description": "The principal for the service instance.",
+          "type": "string",
+          "default": ""
+        },
         "user": {
           "description": "The user that runs the Cassandra nodes and owns the Mesos sandbox.",
           "type": "string",
           "default": "core"
+        },
+        "mesos_api_version" : {
+          "description":"Configures the Mesos API version to use. Possible values: V0 (non-HTTP), V1 (HTTP)",
+          "type":"string",
+          "default":"V0"
         }
       },
       "required": [

diff --git a/frameworks/cassandra/universe/marathon.json.mustache b/frameworks/cassandra/universe/marathon.json.mustache
@@ -3,7 +3,7 @@
   "cpus": 1.0,
   "mem": 2048,
   "instances": 1,
-  "cmd": "export LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib && export MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so) && export PATH=$(ls -d $MESOS_SANDBOX/jre*/bin):$PATH && ./cassandra-scheduler/bin/cassandra ./cassandra-scheduler/svc.yml",
+  "cmd": "{{service.cmd_prefix}} && ./cassandra-scheduler/bin/cassandra ./cassandra-scheduler/svc.yml",
   "labels": {
     "DCOS_PACKAGE_FRAMEWORK_NAME": "{{service.name}}",
     "DCOS_MIGRATION_API_VERSION": "v1",
@@ -13,10 +13,26 @@
     "DCOS_SERVICE_PORT_INDEX": "0",
     "DCOS_SERVICE_SCHEME": "http"
   },
+  {{#service.secret_name}}
+  "secrets": {
+    "serviceCredential": {
+      "source": "{{service.secret_name}}"
+    }
+  },
+  {{/service.secret_name}}
   "env": {
     "FRAMEWORK_NAME": "{{service.name}}",
     "SERVICE_USER": "{{service.user}}",
+    "FRAMEWORK_PRINCIPAL": "{{service.principal}}",
     "CASSANDRA_VERSION": "3.0.10",
+
+    {{#service.secret_name}}
+    "DCOS_SERVICE_ACCOUNT_CREDENTIAL": { "secret": "serviceCredential" },
+    "MESOS_MODULES": "{\"libraries\": [{\"file\": \"libdcos_security.so\", \"modules\": [{\"name\": \"com_mesosphere_dcos_ClassicRPCAuthenticatee\"}]}]}",
+    "MESOS_AUTHENTICATEE": "com_mesosphere_dcos_ClassicRPCAuthenticatee",
+    {{/service.secret_name}}
+    "MESOS_API_VERSION": "{{service.mesos_api_version}}",
+
     "TASKCFG_ALL_CASSANDRA_CLUSTER_NAME": "{{cassandra.cluster_name}}",
     "TASKCFG_ALL_CASSANDRA_AUTHENTICATOR": "{{cassandra.authenticator}}",
     "TASKCFG_ALL_CASSANDRA_AUTHORIZER": "{{cassandra.authorizer}}",

diff --git a/frameworks/cassandra/universe/resource.json b/frameworks/cassandra/universe/resource.json
@@ -2,7 +2,7 @@
   "assets": {
     "uris": {
       "jre-tar-gz": "https://downloads.mesosphere.com/java/jre-8u121-linux-x64.tar.gz",
-      "libmesos-bundle-tar-gz": "http://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-2.tar.gz",
+      "libmesos-bundle-tar-gz": "https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-3.tar.gz",
       "scheduler-zip": "{{artifact-dir}}/cassandra-scheduler.zip",
       "bootstrap-zip": "{{artifact-dir}}/bootstrap.zip",
       "executor-zip": "{{artifact-dir}}/executor.zip"

diff --git a/frameworks/elastic/src/main/dist/elastic_service.yml b/frameworks/elastic/src/main/dist/elastic_service.yml
@@ -1,5 +1,8 @@
 name: {{FRAMEWORK_NAME}}
 web-url: http://proxylite-0-server.{{FRAMEWORK_NAME}}.mesos:{{PROXYLITE_PORT}}
+scheduler:
+  principal: {{FRAMEWORK_PRINCIPAL}}
+  user: {{FRAMEWORK_USER}}
 pods:
   kibana:
     count: {{KIBANA_COUNT}}

diff --git a/frameworks/elastic/universe/config.json b/frameworks/elastic/universe/config.json
@@ -10,11 +10,26 @@
           "type": "string",
           "default": "elastic"
         },
+        "secret_name": {
+          "description":"Name of the Secret Store credentials to use for DC/OS service authentication. This should be left empty unless service authentication is needed.",
+          "type":"string",
+          "default":""
+        },
         "user": {
           "description": "The user that runs the Elasticsearch/Kibana services and owns the Mesos sandbox.",
           "type": "string",
           "default": "core"
         },
+        "principal": {
+          "description": "The principal for the Elasticsearch service instance.",
+          "type": "string",
+          "default": ""
+        },
+        "cmd_prefix" : {
+          "description":"A generic prefix to start the scheduler.",
+          "type":"string",
+          "default":"export LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib:$LD_LIBRARY_PATH; export MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so); export JAVA_HOME=$(ls -d $MESOS_SANDBOX/jre*/); export JAVA_HOME=${JAVA_HOME%/}; export PATH=$(ls -d $JAVA_HOME/bin):$PATH"
+        },
         "plan_strategy": {
           "description": "Elasticsearch and Kibana rollout strategy. [parallel, serial]",
           "type": "string",

diff --git a/frameworks/elastic/universe/marathon.json.mustache b/frameworks/elastic/universe/marathon.json.mustache
@@ -3,13 +3,20 @@
   "cpus": 0.5,
   "mem": 2048,
   "instances": 1,
-  "cmd": "export LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib && export MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so) && export PATH=$(ls -d $MESOS_SANDBOX/jre*/bin):$PATH && env && ./elastic-scheduler/bin/elastic ./elastic-scheduler/elastic_service.yml",
+  "cmd": "{{service.cmd_prefix}} && env && ./elastic-scheduler/bin/elastic ./elastic-scheduler/elastic_service.yml",
   "labels": {
     "DCOS_PACKAGE_FRAMEWORK_NAME": "{{service.name}}",
     "DCOS_MIGRATION_API_VERSION": "v1",
     "DCOS_MIGRATION_API_PATH": "/v1/plan",
     "MARATHON_SINGLE_INSTANCE_APP": "true"
   },
+  {{#service.secret_name}}
+  "secrets": {
+    "serviceCredential": {
+      "source": "{{service.secret_name}}"
+    }
+  },
+  {{/service.secret_name}}
   "env": {
     "ELASTIC_VERSION": "5.1.2",
     "EXECUTOR_URI": "{{resource.assets.uris.executor-zip}}",
@@ -18,6 +25,14 @@
     "LIBMESOS_URI": "{{resource.assets.uris.libmesos-bundle-tar-gz}}",
     "FRAMEWORK_NAME": "{{service.name}}",
     "FRAMEWORK_USER": "{{service.user}}",
+    "FRAMEWORK_PRINCIPAL": "{{service.principal}}",
+
+    {{#service.secret_name}}
+    "DCOS_SERVICE_ACCOUNT_CREDENTIAL": { "secret": "serviceCredential" },
+    "MESOS_MODULES": "{\"libraries\": [{\"file\": \"libdcos_security.so\", \"modules\": [{\"name\": \"com_mesosphere_dcos_ClassicRPCAuthenticatee\"}]}]}",
+    "MESOS_AUTHENTICATEE": "com_mesosphere_dcos_ClassicRPCAuthenticatee",
+    {{/service.secret_name}}
+
     "PLAN_STRATEGY": "{{service.plan_strategy}}",
     "ELASTICSEARCH_PLUGINS": "{{elasticsearch.plugins}}",
     "MASTER_NODE_CPUS": "{{master_nodes.cpus}}",

diff --git a/frameworks/elastic/universe/resource.json b/frameworks/elastic/universe/resource.json
@@ -2,7 +2,7 @@
   "assets": {
     "uris": {
       "jre-tar-gz": "https://downloads.mesosphere.com/java/jre-8u121-linux-x64.tar.gz",
-      "libmesos-bundle-tar-gz": "https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-2.tar.gz",
+      "libmesos-bundle-tar-gz": "https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-3.tar.gz",
       "executor-zip": "{{artifact-dir}}/executor.zip",
       "bootstrap-zip": "{{artifact-dir}}/bootstrap.zip",
       "scheduler-zip": "{{artifact-dir}}/elastic-scheduler.zip"

diff --git a/frameworks/hdfs/universe/config.json b/frameworks/hdfs/universe/config.json
@@ -20,10 +20,15 @@
           "type":"string",
           "default":"hdfs-principal"
         },
+        "secret_name": {
+          "description":"Name of the Secret Store credentials to use for DC/OS service authentication. This should be left empty unless service authentication is needed.",
+          "type":"string",
+          "default":""
+        },
         "cmd_prefix" : {
           "description":"A generic prefix to start the scheduler.",
           "type":"string",
-          "default":"LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib:$LD_LIBRARY_PATH MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so) PATH=$(ls -d $MESOS_SANDBOX/jre*/bin):$PATH"
+          "default":"export LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib:$LD_LIBRARY_PATH; export MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so); export JAVA_HOME=$(ls -d $MESOS_SANDBOX/jre*/); export JAVA_HOME=${JAVA_HOME%/}; export PATH=$(ls -d $JAVA_HOME/bin):$PATH"
         },
         "spec_file" : {
           "description":"The name of the service spec yaml file.",

diff --git a/frameworks/hdfs/universe/marathon.json.mustache b/frameworks/hdfs/universe/marathon.json.mustache
@@ -4,7 +4,6 @@
   "mem": 2048,
   "instances": 1,
   "cmd": "{{service.cmd_prefix}} && ./hdfs-scheduler/bin/hdfs ./hdfs-scheduler/{{service.spec_file}}",
-  "user": "{{service.user}}",
   "labels": {
     "DCOS_PACKAGE_FRAMEWORK_NAME": "{{service.name}}",
     "DCOS_MIGRATION_API_VERSION": "v1",
@@ -14,6 +13,13 @@
     "DCOS_SERVICE_PORT_INDEX": "0",
     "DCOS_SERVICE_SCHEME": "http"
   },
+  {{#service.secret_name}}
+  "secrets": {
+    "serviceCredential": {
+      "source": "{{service.secret_name}}"
+    }
+  },
+  {{/service.secret_name}}
   "env": {
     "FRAMEWORK_NAME": "{{service.name}}",
     "HDFS_VERSION": "hadoop-2.6.0-cdh5.9.1",
@@ -40,6 +46,13 @@
     "LIBMESOS_URI": "{{resource.assets.uris.libmesos-bundle-tar-gz}}",
     "HDFS_URI": "{{resource.assets.uris.hdfs-tar-gz}}",
     "BOOTSTRAP_URI": "{{resource.assets.uris.bootstrap-zip}}",
+
+    {{#service.secret_name}}
+    "DCOS_SERVICE_ACCOUNT_CREDENTIAL": { "secret": "serviceCredential" },
+    "MESOS_MODULES": "{\"libraries\": [{\"file\": \"libdcos_security.so\", \"modules\": [{\"name\": \"com_mesosphere_dcos_ClassicRPCAuthenticatee\"}]}]}",
+    "MESOS_AUTHENTICATEE": "com_mesosphere_dcos_ClassicRPCAuthenticatee",
+    {{/service.secret_name}}
+
     {{#service.kerberos.enabled}}
     "KEYTABS_URI": "{{service.kerberos.keytabs_uri}}",
     "KRB5_CONF_URI": "{{service.kerberos.krb5_conf_uri}}",
@@ -48,6 +61,7 @@
     "TASKCFG_ALL_KERBEROS_PRIMARY_HTTP": "{{service.kerberos.primary_http}}",
     "TASKCFG_ALL_KERBEROS_REALM": "{{service.kerberos.realm}}",
     {{/service.kerberos.enabled}}
+
     "TASKCFG_ALL_NAME_NODE_RPC_PORT":"{{hdfs.name_node_rpc_port}}",
     "TASKCFG_ALL_NAME_NODE_HTTP_PORT":"{{hdfs.name_node_http_port}}",
     "TASKCFG_ALL_JOURNAL_NODE_RPC_PORT":"{{hdfs.journal_node_rpc_port}}",

diff --git a/frameworks/hdfs/universe/resource.json b/frameworks/hdfs/universe/resource.json
@@ -2,7 +2,7 @@
   "assets": {
     "uris": {
       "jre-tar-gz": "https://downloads.mesosphere.com/java/jre-8u112-linux-x64-jce-unlimited.tar.gz",
-      "libmesos-bundle-tar-gz": "https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-2.tar.gz",
+      "libmesos-bundle-tar-gz": "https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-3.tar.gz",
       "hdfs-tar-gz": "https://downloads.mesosphere.com/hdfs/assets/hadoop-2.6.0-cdh5.9.1-dcos.tar.gz",
       "bootstrap-zip": "{{artifact-dir}}/bootstrap.zip",
       "scheduler-zip": "{{artifact-dir}}/hdfs-scheduler.zip",

diff --git a/frameworks/helloworld/src/main/dist/svc.yml b/frameworks/helloworld/src/main/dist/svc.yml
@@ -1,4 +1,7 @@
 name: hello-world
+scheduler:
+  principal: {{SERVICE_PRINCIPAL}}
+  user: {{SERVICE_USER}}
 pods:
   hello:
     count: {{HELLO_COUNT}}

diff --git a/frameworks/helloworld/universe/config.json b/frameworks/helloworld/universe/config.json
@@ -28,7 +28,22 @@
           "cmd_prefix" : {
             "description":"A generic prefix to start the scheduler.",
             "type":"string",
-            "default":"LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib:$LD_LIBRARY_PATH MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so) PATH=$(ls -d $MESOS_SANDBOX/jre*/bin):$PATH"
+            "default":"export LD_LIBRARY_PATH=$MESOS_SANDBOX/libmesos-bundle/lib:$LD_LIBRARY_PATH; export MESOS_NATIVE_JAVA_LIBRARY=$(ls $MESOS_SANDBOX/libmesos-bundle/lib/libmesos-*.so); export JAVA_HOME=$(ls -d $MESOS_SANDBOX/jre*/); export JAVA_HOME=${JAVA_HOME%/}; export PATH=$(ls -d $JAVA_HOME/bin):$PATH"
+          },
+          "secret_name": {
+            "description":"Name of the Secret Store credentials to use for DC/OS service authentication. This should be left empty unless service authentication is needed.",
+            "type":"string",
+            "default":""
+          },
+          "user": {
+            "description": "The user that the service will run as.",
+            "type": "string",
+            "default": "root"
+          },
+          "principal": {
+            "description": "The principal for the service instance.",
+            "type": "string",
+            "default": ""
           }
         }
       },

diff --git a/frameworks/helloworld/universe/marathon.json.mustache b/frameworks/helloworld/universe/marathon.json.mustache
@@ -13,8 +13,17 @@
     "DCOS_SERVICE_PORT_INDEX": "0",
     "DCOS_SERVICE_SCHEME": "http"
   },
+  {{#service.secret_name}}
+  "secrets": {
+    "serviceCredential": {
+      "source": "{{service.secret_name}}"
+    }
+  },
+  {{/service.secret_name}}
   "env": {
     "FRAMEWORK_NAME": "{{service.name}}",
+    "SERVICE_PRINCIPAL": "{{service.principal}}",
+    "SERVICE_USER": "{{service.user}}",
     "HELLO_COUNT": "{{hello.count}}",
     "HELLO_CPUS": "{{hello.cpus}}",
     "HELLO_MEM": "{{hello.mem}}",
@@ -28,6 +37,11 @@
     "SLEEP_DURATION": "{{service.sleep}}",
     "EXECUTOR_URI": "{{resource.assets.uris.executor-zip}}",
     "LIBMESOS_URI": "{{resource.assets.uris.libmesos-bundle-tar-gz}}",
+    {{#service.secret_name}}
+    "DCOS_SERVICE_ACCOUNT_CREDENTIAL": { "secret": "serviceCredential" },
+    "MESOS_MODULES": "{\"libraries\": [{\"file\": \"libdcos_security.so\", \"modules\": [{\"name\": \"com_mesosphere_dcos_ClassicRPCAuthenticatee\"}]}]}",
+    "MESOS_AUTHENTICATEE": "com_mesosphere_dcos_ClassicRPCAuthenticatee",
+    {{/service.secret_name}}
     "MESOS_API_VERSION": "{{service.mesos_api_version}}"
   },
   "uris": [

diff --git a/frameworks/helloworld/universe/resource.json b/frameworks/helloworld/universe/resource.json
@@ -2,7 +2,7 @@
   "assets": {
     "uris": {
       "jre-tar-gz": "https://downloads.mesosphere.com/java/jre-8u112-linux-x64.tar.gz",
-      "libmesos-bundle-tar-gz": "https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-2.tar.gz",
+      "libmesos-bundle-tar-gz": "https://downloads.mesosphere.com/libmesos-bundle/libmesos-bundle-1.9-argus-1.1.x-3.tar.gz",
       "scheduler-zip": "{{artifact-dir}}/hello-world-scheduler.zip",
       "executor-zip": "{{artifact-dir}}/executor.zip"
     }

diff --git a/frameworks/kafka/src/main/dist/svc.yml b/frameworks/kafka/src/main/dist/svc.yml
@@ -1,8 +1,7 @@
 name: {{FRAMEWORK_NAME}}
 scheduler:
-  principal: {{FRAMEWORK_PRINCIPLE}}
-  zookeeper: {{MESOS_ZOOKEEPER_URI}}
-  api-port: {{PORT_API}}
+  principal: {{FRAMEWORK_PRINCIPAL}}
+  user: {{FRAMEWORK_USER}}
 pods:
   kafka:
     count: {{BROKER_COUNT}}
@@ -27,6 +26,8 @@ pods:
           path: {{BROKER_DATA_PATH}}
           type: {{BROKER_DISK_TYPE}}
           size: {{BROKER_DISK_SIZE}}
+        env:
+          KAFKA_ZOOKEEPER_URI: "{{MESOS_ZOOKEEPER_URI}}/dcos-service-{{FRAMEWORK_NAME}}"
         goal: RUNNING
         cmd: "./bootstrap -resolve=false && exec $MESOS_SANDBOX/{{KAFKA_VERSION_PATH}}/bin/kafka-server-start.sh $MESOS_SANDBOX/{{KAFKA_VERSION_PATH}}/config/server.properties"
         configs: