Skip to content

Commit

Permalink
Allow search templates stored in an index to be retrieved and used at…
Browse files Browse the repository at this point in the history 
… search time

This change allows search templates stored in an index to be used at search time.
You can run an indexed template by
GET /_search/template
{
    'templatename' : '/index/language/id',
    'params' : {
        'param1' : 'foo'
    }
}
Running the template will fail if the template cannot be found, if the templatename
starts with a '/' but does not conform to the /index/type/id triple
If the named template cannot be found an error will be raised. Currently the
templates are NOT cached. We also use TransportGetAction directly instead of using
Client to avoid a circular dependency between Client and ScriptService

See elastic#5637
  • Loading branch information
@GaelTadh
GaelTadh committed Jul 14, 2014
1 parent dcb2107 commit eafb3b1
Show file tree
Hide file tree
Showing 2 changed files with 191 additions and 3 deletions.
130 changes: 127 additions & 3 deletions src/main/java/org/elasticsearch/script/ScriptService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.TransportGetAction;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
Expand All @@ -35,6 +40,7 @@
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.watcher.FileChangesListener;
import org.elasticsearch.watcher.FileWatcher;
Expand All @@ -49,6 +55,11 @@
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import static org.elasticsearch.client.Requests.getRequest;

/**
*
Expand Down Expand Up @@ -157,7 +168,76 @@ public CompiledScript compile(String script) {
return compile(defaultLang, script);
}


private class ScriptResponse implements ActionListener<GetResponse>{
public String localScript = null;
final Lock lock = new ReentrantLock();
public final Condition gotResponseCondition = lock.newCondition();

@Override
public void onResponse(GetResponse getFields) {
lock.lock();
try {
logger.warn("Got script response " + getFields.toString());
if (getFields.isExists()) {
localScript = getFields.getSourceAsString();
logger.warn("Localscript set to " + localScript);
}
else {
logger.warn("Got response but does not exist");
}
} finally {
gotResponseCondition.signalAll();
lock.unlock();
}
}

@Override
public void onFailure(Throwable e) {
lock.lock();
try {
logger.warn("Failed to get script response", e);
localScript = null;
} finally {
gotResponseCondition.signalAll();
lock.unlock();
}
}
}

public CompiledScript compile(String lang, String script) {
CacheKey cacheKey = new CacheKey(lang, script);

String scriptContent = null;

if(script.startsWith("/")){ //This is how we determine if we need to search the index for the script
if( getAction == null ){
throw new ElasticsearchIllegalArgumentException("Got an indexed script with no TransportGetAction registered.");
}
String[] parts = script.split("/");
if (parts.length != 4) {
throw new ElasticsearchIllegalArgumentException("Illegal index script format [" + script + "]" +
" should be /index/lang/id" );
} else {
final String index = parts[1];
final String scriptLang = parts[2];
final String id = parts[3];
if (lang != null && !lang.equals(scriptLang)){
logger.trace("Overriding lang to " + scriptLang);
lang = scriptLang;
//cacheKey = new CacheKey(lang,script);
}
/*
compiled = cache.getIfPresent(cacheKey);
if (compiled != null) {
return compiled;
}
*/
scriptContent = getScriptFromIndex(script, index, scriptLang, id);
}
}

CompiledScript compiled = staticCache.get(script);
if (compiled != null) {
return compiled;
Expand All @@ -168,7 +248,7 @@ public CompiledScript compile(String lang, String script) {
if (!dynamicScriptEnabled(lang)) {
throw new ScriptException("dynamic scripting for [" + lang + "] disabled");
}
CacheKey cacheKey = new CacheKey(lang, script);

compiled = cache.getIfPresent(cacheKey);
if (compiled != null) {
return compiled;
Expand All @@ -178,11 +258,55 @@ public CompiledScript compile(String lang, String script) {
if (service == null) {
throw new ElasticsearchIllegalArgumentException("script_lang not supported [" + lang + "]");
}
compiled = new CompiledScript(lang, service.compile(script));
cache.put(cacheKey, compiled);
if (scriptContent != null) {
compiled = new CompiledScript(lang, service.compile(scriptContent)); //We have loaded the script from the index
} else {
compiled = new CompiledScript(lang, service.compile(script));
cache.put(cacheKey, compiled); //only cache non indexed templates for now
}
return compiled;
}

private String getScriptFromIndex(String script, String index, String scriptLang, String id) {
GetRequest getScriptRequest = getRequest(index)
.fields(TEMPLATE_GET_FIELDS)
.type(scriptLang)
.id(id)
.listenerThreaded(true)
.operationThreaded(true);

ScriptResponse scriptResponse = new ScriptResponse();

getAction.execute(getScriptRequest, scriptResponse);

String timeout = settings.get("template.index.lookup.timeout", "10000");

Date deadline = new Date(System.currentTimeMillis() + Long.parseLong(timeout));

scriptResponse.lock.lock();
try {
while (true) {
try {
if (!scriptResponse.gotResponseCondition.awaitUntil(deadline)) {
throw new ElasticsearchTimeoutException("Timed out attempting to read template " + script);
}
if (scriptResponse.localScript != null) {
script = scriptResponse.localScript; //Get the script from the response
} else {
throw new ElasticsearchIllegalArgumentException("Unable to find script [" + script + "]");
}
break;
} catch (InterruptedException ie) {
continue;
}
}
} finally {
scriptResponse.lock.unlock();
}
return script;
}


public ExecutableScript executable(String lang, String script, Map vars) {
return executable(compile(lang, script), vars);
}
Expand Down
64 changes: 64 additions & 0 deletions src/test/java/org/elasticsearch/index/query/TemplateQueryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
package org.elasticsearch.index.query;

import com.google.common.collect.Maps;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.bytes.BytesArray;
Expand All @@ -29,11 +31,15 @@
import org.junit.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -213,4 +219,62 @@ public void testThatParametersCanBeSet() throws Exception {
searchResponse = client().prepareSearch("test").setTypes("type").setTemplateName("full-query-template").setTemplateParams(templateParams).get();
assertHitCount(searchResponse, 1);
}

@Test
public void testIndexedTemplate() throws Exception {
createIndex("template_index");
ensureGreen("template_index");
List<IndexRequestBuilder> builders = new ArrayList<IndexRequestBuilder>();
builders.add(client().prepareIndex("template_index","mustache","1").setSource("{" +
" \"query\":{" +
" \"match\":{" +
" \"theField\" : \"{{fieldParam}}\"}" +
" }" +
"}"));
indexRandom(true, builders);

builders.clear();

builders.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
builders.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
builders.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}"));
builders.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}"));
builders.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}"));

indexRandom(true,builders);

Map<String, String> templateParams = Maps.newHashMap();
templateParams.put("fieldParam", "foo");

SearchResponse searchResponse = client().prepareSearch("test").setTypes("type").
setTemplateName("/template_index/mustache/1").setTemplateParams(templateParams).get();
assertHitCount(searchResponse, 4);

Exception e = null;
try {
searchResponse = client().prepareSearch("test").setTypes("type").
setTemplateName("/template_index/mustache/1000").setTemplateParams(templateParams).get();
} catch (SearchPhaseExecutionException spee) {
e = spee;
}
assert e != null;
e = null;
try {
searchResponse = client().prepareSearch("test").setTypes("type").
setTemplateName("/template_indexmustache/1").setTemplateParams(templateParams).get();
assertFailures(searchResponse);
} catch (SearchPhaseExecutionException spee){
e = spee;
}
assert e != null;



templateParams.put("fieldParam", "bar");
searchResponse = client().prepareSearch("test").setTypes("type").
setTemplateName("/template_index/mustache/1").setTemplateParams(templateParams).get();
assertHitCount(searchResponse, 1);
}


}

0 comments on commit eafb3b1

Please sign in to comment.