Add support for fine-grained settings

dev-tools/build_release.py

@@ -437,7 +437,7 @@ def smoke_test_release(release, files, expected_hash, plugins):
     else:
       background = '-d'
     print('  Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release))
-    run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.discovery.zen.ping.multicast.enabled=false -Des.script.disable_dynamic=false %s'
+    run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.discovery.zen.ping.multicast.enabled=false -Des.script.inline=on -Des.script.indexed=on %s'
          % (java_exe(), es_run_path, background))
     conn = HTTPConnection('127.0.0.1', 9200, 20);
     wait_for_node_startup()

dev-tools/create-bwc-index.py

@@ -107,7 +107,8 @@ def start_node(version, release_dir, data_dir, tcp_port, http_port):
     '-Des.cluster.name=bwc_index_' + version,  
     '-Des.network.host=localhost', 
     '-Des.discovery.zen.ping.multicast.enabled=false',
-    '-Des.script.disable_dynamic=true',
+    '-Des.script.inline=on',
+    '-Des.script.indexed=on',
     '-Des.transport.tcp.port=%s' % tcp_port,
     '-Des.http.port=%s' % http_port
   ]

dev-tools/upgrade-tests.py

@@ -107,7 +107,8 @@ def start_node(version, data_dir, node_dir, unicast_host_list, tcp_port, http_po
     '-Des.path.data=%s' % data_dir, '-Des.cluster.name=upgrade_test',  
     '-Des.discovery.zen.ping.unicast.hosts=%s' % unicast_host_list, 
     '-Des.discovery.zen.ping.multicast.enabled=false',
-    '-Des.script.disable_dynamic=true',
+    '-Des.script.inline=on',
+    '-Des.script.indexed=on',
     '-Des.transport.tcp.port=%s' % tcp_port,
     '-Des.http.port=%s' % http_port,
     foreground], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

docs/reference/modules/scripting.asciidoc

@@ -145,9 +145,9 @@ named `group1_group2_test`.
 
 [float]
 === Indexed Scripts
-If dynamic scripting is enabled, Elasticsearch allows you to store scripts
-in an internal index known as `.scripts` and reference them by id. There are
-REST endpoints to manage indexed scripts as follows:
+Elasticsearch allows you to store scripts in an internal index known as
+`.scripts` and reference them by id. There are REST endpoints to manage
+indexed scripts as follows:
 
 Requests to the scripts endpoint look like :
 [source,js]
@@ -196,8 +196,6 @@ curl -XPOST localhost:9200/_search -d '{
   }
 }'
 --------------------------------------------------
-Note that you must have dynamic scripting enabled to use indexed scripts
-for non sandboxed languages at query time.
 
 The script can be viewed by:
 [source,js]
@@ -221,42 +219,117 @@ curl -XDELETE localhost:9200/_scripts/groovy/indexedCalculateScore
 -----------------------------------
 
 [float]
+[[enable-dynamic-scripting]]
 === Enabling dynamic scripting
 
 We recommend running Elasticsearch behind an application or proxy, which
 protects Elasticsearch from the outside world. If users are allowed to run
-dynamic scripts (even in a search request), then they have the same access to
-your box as the user that Elasticsearch is running as. For this reason dynamic
-scripting is allowed only for sandboxed languages by default.
+inline scripts (even in a search request) or indexed scripts, then they have
+the same access to your box as the user that Elasticsearch is running as. For
+this reason dynamic scripting is allowed only for sandboxed languages by default.
+The script engines that currently support sandboxing are `mustache` and
+`expressions`.
 
 First, you should not run Elasticsearch as the `root` user, as this would allow
 a script to access or do *anything* on your server, without limitations. Second,
 you should not expose Elasticsearch directly to users, but instead have a proxy
 application inbetween. If you *do* intend to expose Elasticsearch directly to
 your users, then you have to decide whether you trust them enough to run scripts
-on your box or not. If you do, you can enable dynamic scripting by adding the
-following setting to the `config/elasticsearch.yml` file on every node:
+on your box or not.
+
+deprecated[1.6.0, the `script.disable_dynamic` setting is deprecated in favour of fine-grained settings described as follows]
+
+coming[1.6.0, Fine-grained script settings replace the `script.disable_dynamic` setting]
+
+It is possible to enable scripts based on their source, for
+every script engine, through the following settings that need to be added to the
+`config/elasticsearch.yml` file on every node.
 
 [source,yaml]
 -----------------------------------
-script.disable_dynamic: false
+script.inline: on
+script.indexed: on
+
 -----------------------------------
 
 While this still allows execution of named scripts provided in the config, or
 _native_ Java scripts registered through plugins, it also allows users to run
 arbitrary scripts via the API. Instead of sending the name of the file as the
-script, the body of the script can be sent instead.
+script, the body of the script can be sent instead or retrieved from the
+`.scripts` indexed if previously stored.
 
-There are three possible configuration values for the `script.disable_dynamic`
-setting, the default value is `sandbox`:
+There are three possible configuration values for any of the fine-grained
+script settings:
 
 [cols="<,<",options="header",]
 |=======================================================================
 |Value |Description
-| `true` |all dynamic scripting is disabled, scripts must be placed in the `config/scripts` directory.
-| `false` |all dynamic scripting is enabled, scripts may be sent as strings in requests.
-| `sandbox` |scripts may be sent as strings for languages that are sandboxed.
+| `off` |scripting is turned off completely, in the context of the setting being set.
+| `on`  |scripting is turned on, in the context of the setting being set.
+| `sandbox` |scripts may be executed only for languages that are sandboxed (`mustache` and `expressions`)
+|=======================================================================
+
+The default values are the following:
+
+[source,yaml]
+-----------------------------------
+script.inline: sandbox
+script.indexed: sandbox
+script.file: on
+
+-----------------------------------
+
+NOTE: Global scripting settings affect the `mustache` scripting language.
+<<search-template,Search templates>> internally use the `mustache` language,
+and will still be enabled by default as the `mustache` engine is sandboxed,
+but they will be enabled/disabled according to fine-grained settings
+specified in `elasticsearch.yml`. coming[1.6.0, `mustache` scripts were previously always on regardless of whether dynamic scripts were enabled or not]
+
+It is also possible to control which operations can execute scripts. The
+supported operations are:
+
+[cols="<,<",options="header",]
 |=======================================================================
+|Value |Description
+| `aggs`    |Aggregations (wherever they may be used)
+| `mapping` |Mappings (script transform feature)
+| `search`  |Search api, Percolator api and Suggester api (e.g filters, script_fields)
+| `update`  |Update api
+|=======================================================================
+
+The following example disables scripting for `update` and `mapping` operations,
+regardless of the script source, for any engine. Scripts can still be
+executed from sandboxed languages as part of `aggregations` and `search`
+operations though, as the above defaults still get applied.
+
+[source,yaml]
+-----------------------------------
+script.update: off
+script.mapping: off
+
+-----------------------------------
+
+Generic settings get applied in order, operation based ones have precedence
+over source based ones. Language specific settings are supported too. They
+need to be prefixed with the `script.engine.<engine>` prefix and have
+precedence over any other generic settings.
+
+[source,yaml]
+-----------------------------------
+script.engine.groovy.file.aggs: on
+script.engine.groovy.file.mapping: on
+script.engine.groovy.file.search: on
+script.engine.groovy.file.update: on
+script.engine.groovy.indexed.aggs: on
+script.engine.groovy.indexed.mapping: off
+script.engine.groovy.indexed.search: on
+script.engine.groovy.indexed.update: off
+script.engine.groovy.inline.aggs: on
+script.engine.groovy.inline.mapping: off
+script.engine.groovy.inline.search: off
+script.engine.groovy.inline.update: off
+
+-----------------------------------
 
 [float]
 === Default Scripting Language

docs/reference/search/search-template.asciidoc

@@ -25,6 +25,11 @@ For more information on how Mustache templating and what kind of templating you
 can do with it check out the http://mustache.github.io/mustache.5.html[online
 documentation of the mustache project].
 
+NOTE: The mustache language is implemented in elasticsearch as a sandboxed
+scripting language, hence it obeys settings that may be used to enable or
+disable scripts per language, source and operation as described in
+<<enable-dynamic-scripting, scripting docs>> coming[1.6.0, `mustache` scripts were always on before and it wasn't possible to disable them].
+
 [float]
 ==== More template examples
 
@@ -296,4 +301,4 @@ GET /_search/template
     }
 }
 ------------------------------------------
-<1> Name of the the query template stored in the .scripts index.
+<1> Name of the the query template stored in the `.scripts` index.

src/main/java/org/elasticsearch/action/update/UpdateHelper.java

@@ -46,6 +46,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
@@ -93,7 +94,7 @@ public Result prepare(UpdateRequest request, IndexShard indexShard) {
                 ctx.put("op", "create");
                 ctx.put("_source", upsertDoc);
                 try {
-                    ExecutableScript script = scriptService.executable(request.scriptLang, request.script, request.scriptType, request.scriptParams);
+                    ExecutableScript script = scriptService.executable(request.scriptLang, request.script, request.scriptType, ScriptContext.UPDATE, request.scriptParams);
                     script.setNextVar("ctx", ctx);
                     script.run();
                     // we need to unwrap the ctx...
@@ -191,7 +192,7 @@ public Result prepare(UpdateRequest request, IndexShard indexShard) {
             ctx.put("_source", sourceAndContent.v2());
 
             try {
-                ExecutableScript script = scriptService.executable(request.scriptLang, request.script, request.scriptType, request.scriptParams);
+                ExecutableScript script = scriptService.executable(request.scriptLang, request.script, request.scriptType, ScriptContext.UPDATE, request.scriptParams);
                 script.setNextVar("ctx", ctx);
                 script.run();
                 // we need to unwrap the ctx...

src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -45,7 +45,6 @@
 import org.elasticsearch.common.text.StringAndBytesText;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.*;
-import org.elasticsearch.common.xcontent.smile.SmileXContent;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.cache.fixedbitset.FixedBitSetFilterCache;
 import org.elasticsearch.index.mapper.internal.*;
@@ -54,6 +53,7 @@
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.ScriptContext;
 
 import java.io.IOException;
 import java.util.*;
@@ -845,7 +845,7 @@ public ScriptTransform(ScriptService scriptService, String script, ScriptType sc
         public Map<String, Object> transformSourceAsMap(Map<String, Object> sourceAsMap) {
             try {
                 // We use the ctx variable and the _source name to be consistent with the update api.
-                ExecutableScript executable = scriptService.executable(language, script, scriptType, parameters);
+                ExecutableScript executable = scriptService.executable(language, script, scriptType, ScriptContext.MAPPING, parameters);
                 Map<String, Object> ctx = new HashMap<>(1);
                 ctx.put("_source", sourceAsMap);
                 executable.setNextVar("ctx", ctx);

src/main/java/org/elasticsearch/index/query/ScriptFilterParser.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
 import org.elasticsearch.script.ScriptParameterParser;
+import org.elasticsearch.script.*;
 import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
@@ -132,7 +133,7 @@ public static class ScriptFilter extends Filter {
         public ScriptFilter(String scriptLang, String script, ScriptService.ScriptType scriptType, Map<String, Object> params, ScriptService scriptService, SearchLookup searchLookup) {
             this.script = script;
             this.params = params;
-            this.searchScript = scriptService.search(searchLookup, scriptLang, script, scriptType, newHashMap(params));
+            this.searchScript = scriptService.search(searchLookup, scriptLang, script, scriptType, ScriptContext.SEARCH, newHashMap(params));
         }
 
         @Override

src/main/java/org/elasticsearch/index/query/TemplateQueryParser.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
 
 import java.io.IOException;
@@ -76,7 +77,7 @@ public String[] names() {
     public Query parse(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         TemplateContext templateContext = parse(parser, PARAMS, parametersToTypes);
-        ExecutableScript executable = this.scriptService.executable(MustacheScriptEngineService.NAME, templateContext.template(), templateContext.scriptType(), templateContext.params());
+        ExecutableScript executable = this.scriptService.executable(MustacheScriptEngineService.NAME, templateContext.template(), templateContext.scriptType(), ScriptContext.SEARCH, templateContext.params());
 
         BytesReference querySource = (BytesReference) executable.run();
 

src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionParser.java

@@ -28,7 +28,7 @@
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryParsingException;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
-import org.elasticsearch.script.ScriptParameterParser;
+import org.elasticsearch.script.*;
 import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
@@ -87,7 +87,7 @@ public ScoreFunction parse(QueryParseContext parseContext, XContentParser parser
 
         SearchScript searchScript;
         try {
-            searchScript = parseContext.scriptService().search(parseContext.lookup(), scriptParameterParser.lang(), script, scriptType, vars);
+            searchScript = parseContext.scriptService().search(parseContext.lookup(), scriptParameterParser.lang(), script, scriptType, ScriptContext.SEARCH, vars);
             return new ScriptScoreFunction(script, vars, searchScript);
         } catch (Exception e) {
             throw new QueryParsingException(parseContext.index(), NAMES[0] + " the script could not be loaded", e);

src/main/java/org/elasticsearch/script/ScriptContext.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import java.util.Locale;
+
+/**
+ * Operation/api that uses a script as part of its execution.
+ * Note that the suggest api is considered part of search for simplicity, as well as the percolate api.
+ */
+public enum ScriptContext {
+    MAPPING,
+    UPDATE,
+    SEARCH,
+    AGGS;
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}

src/main/java/org/elasticsearch/script/ScriptMode.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.ElasticsearchIllegalArgumentException;
+import org.elasticsearch.common.Booleans;
+
+import java.util.Locale;
+
+/**
+ * Mode for a specific script, used for script settings.
+ * Defines whether a certain script or catefory of scripts can be executed or not, or whether it can
+ * only be executed by a sandboxed scripting language.
+ */
+enum ScriptMode {
+    ON,
+    OFF,
+    SANDBOX;
+
+    static ScriptMode parse(String input) {
+        input = input.toLowerCase(Locale.ROOT);
+        if (Booleans.isExplicitTrue(input)) {
+            return ON;
+        }
+        if (Booleans.isExplicitFalse(input)) {
+            return OFF;
+        }
+        if (SANDBOX.toString().equals(input)) {
+            return SANDBOX;
+        }
+        throw new ElasticsearchIllegalArgumentException("script mode [" + input + "] not supported");
+    }
+
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}