Skip to content

Commit

Permalink
Merge pull request #12400 from brwe/plug-fetch-sub-phases
Browse files Browse the repository at this point in the history
Make fetch sub phases pluggable
  • Loading branch information
brwe committed Aug 3, 2015
2 parents c5b91b9 + 8a49579 commit 26d51c2
Show file tree
Hide file tree
Showing 16 changed files with 501 additions and 99 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.elasticsearch.action.percolate.PercolateShardRequest;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.cache.recycler.PageCacheRecycler;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.*;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.lease.Releasables;
Expand All @@ -58,7 +59,7 @@
import org.elasticsearch.search.dfs.DfsSearchResult;
import org.elasticsearch.search.fetch.FetchSearchResult;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext;
import org.elasticsearch.search.fetch.FetchSubPhaseContext;
import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
import org.elasticsearch.search.fetch.script.ScriptFieldsContext;
import org.elasticsearch.search.fetch.source.FetchSourceContext;
Expand Down Expand Up @@ -116,6 +117,7 @@ public class PercolateContext extends SearchContext {
private SearchContextAggregations aggregations;
private QuerySearchResult querySearchResult;
private Sort sort;
private final Map<String, FetchSubPhaseContext> subPhaseContexts = new HashMap<>();

public PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, IndexShard indexShard,
IndexService indexService, PageCacheRecycler pageCacheRecycler,
Expand Down Expand Up @@ -282,6 +284,15 @@ public SearchContext aggregations(SearchContextAggregations aggregations) {
return this;
}

@Override
public <SubPhaseContext extends FetchSubPhaseContext> SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory<SubPhaseContext> contextFactory) {
String subPhaseName = contextFactory.getName();
if (subPhaseContexts.get(subPhaseName) == null) {
subPhaseContexts.put(subPhaseName, contextFactory.newContextInstance());
}
return (SubPhaseContext) subPhaseContexts.get(subPhaseName);
}

// Unused:
@Override
public void preProcess() {
Expand Down Expand Up @@ -378,16 +389,6 @@ public void addRescore(RescoreSearchContext rescore) {
throw new UnsupportedOperationException();
}

@Override
public boolean hasFieldDataFields() {
throw new UnsupportedOperationException();
}

@Override
public FieldDataFieldsContext fieldDataFields() {
throw new UnsupportedOperationException();
}

@Override
public boolean hasScriptFields() {
throw new UnsupportedOperationException();
Expand Down
12 changes: 3 additions & 9 deletions core/src/main/java/org/elasticsearch/search/SearchModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.elasticsearch.search.controller.SearchPhaseController;
import org.elasticsearch.search.dfs.DfsPhase;
import org.elasticsearch.search.fetch.FetchPhase;
import org.elasticsearch.search.fetch.FetchSubPhaseModule;
import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.innerhits.InnerHitsFetchSubPhase;
Expand Down Expand Up @@ -63,7 +64,8 @@ public Iterable<? extends Module> spawnModules() {
new HighlightModule(),
new SuggestModule(),
new FunctionScoreModule(),
new AggregationModule());
new AggregationModule(),
new FetchSubPhaseModule());
}

@Override
Expand All @@ -73,14 +75,6 @@ protected void configure() {
bind(SearchPhaseController.class).asEagerSingleton();

bind(FetchPhase.class).asEagerSingleton();
bind(ExplainFetchSubPhase.class).asEagerSingleton();
bind(FieldDataFieldsFetchSubPhase.class).asEagerSingleton();
bind(ScriptFieldsFetchSubPhase.class).asEagerSingleton();
bind(FetchSourceSubPhase.class).asEagerSingleton();
bind(VersionFetchSubPhase.class).asEagerSingleton();
bind(MatchedQueriesFetchSubPhase.class).asEagerSingleton();
bind(HighlightPhase.class).asEagerSingleton();
bind(InnerHitsFetchSubPhase.class).asEagerSingleton();

bind(SearchServiceTransportAction.class).asEagerSingleton();
bind(MoreLikeThisFetchService.class).asEagerSingleton();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchParseElement;
Expand Down Expand Up @@ -83,13 +84,10 @@ public class FetchPhase implements SearchPhase {
private final FetchSubPhase[] fetchSubPhases;

@Inject
public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase,
MatchedQueriesFetchSubPhase matchedQueriesPhase, ExplainFetchSubPhase explainPhase, VersionFetchSubPhase versionPhase,
FetchSourceSubPhase fetchSourceSubPhase, FieldDataFieldsFetchSubPhase fieldDataFieldsFetchSubPhase,
InnerHitsFetchSubPhase innerHitsFetchSubPhase) {
public FetchPhase(Set<FetchSubPhase> fetchSubPhases, InnerHitsFetchSubPhase innerHitsFetchSubPhase) {
innerHitsFetchSubPhase.setFetchPhase(this);
this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, matchedQueriesPhase, explainPhase, highlightPhase,
fetchSourceSubPhase, versionPhase, fieldDataFieldsFetchSubPhase, innerHitsFetchSubPhase};
this.fetchSubPhases = fetchSubPhases.toArray(new FetchSubPhase[fetchSubPhases.size() + 1]);
this.fetchSubPhases[fetchSubPhases.size()] = innerHitsFetchSubPhase;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,4 +114,23 @@ public String getSourcePath(String sourcePath) {
boolean hitsExecutionNeeded(SearchContext context);

void hitsExecute(SearchContext context, InternalSearchHit[] hits);

/**
* This interface is in the fetch phase plugin mechanism.
* Whenever a new search is executed we create a new {@link SearchContext} that holds individual contexts for each {@link org.elasticsearch.search.fetch.FetchSubPhase}.
* Fetch phases that use the plugin mechanism must provide a ContextFactory to the SearchContext that creates the fetch phase context and also associates them with a name.
* See {@link SearchContext#getFetchSubPhaseContext(FetchSubPhase.ContextFactory)}
*/
public interface ContextFactory<SubPhaseContext extends FetchSubPhaseContext> {

/**
* The name of the context.
*/
public String getName();

/**
* Creates a new instance of a FetchSubPhaseContext that holds all information a FetchSubPhase needs to execute on hits.
*/
public SubPhaseContext newContextInstance();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.search.fetch;

/**
* All configuration and context needed by the FetchSubPhase to execute on hits.
* The only required information in this base class is whether or not the sub phase needs to be run at all.
* It can be extended by FetchSubPhases to hold information the phase needs to execute on hits.
* See {@link org.elasticsearch.search.fetch.FetchSubPhase.ContextFactory} and also {@link org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext} for an example.
*/
public class FetchSubPhaseContext {

// This is to store if the FetchSubPhase should be executed at all.
private boolean hitExecutionNeeded = false;

/**
* Set if this phase should be executed at all.
*/
void setHitExecutionNeeded(boolean hitExecutionNeeded) {
this.hitExecutionNeeded = hitExecutionNeeded;
}

/**
* Returns if this phase be executed at all.
*/
public boolean hitExecutionNeeded() {
return hitExecutionNeeded;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.fetch;

import com.google.common.collect.Lists;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.multibindings.Multibinder;
import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.innerhits.InnerHitsFetchSubPhase;
import org.elasticsearch.search.fetch.matchedqueries.MatchedQueriesFetchSubPhase;
import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.source.FetchSourceSubPhase;
import org.elasticsearch.search.fetch.version.VersionFetchSubPhase;
import org.elasticsearch.search.highlight.HighlightPhase;

import java.util.List;

/**
* Module for registering fetch sub phases. Fetch phases are executed when the document is finally
* retrieved from the shard. To implement a new fetch phase one needs to implement the following classes and interfaces
* <p/>
* <ul>
* <li> {@link FetchSubPhaseParseElement} </li>
* <li> {@link FetchSubPhase} </li>
* <li> {@link FetchSubPhaseContext} </li>
* </ul>
* <p/>
* The FetchSubPhase must then be registered with this module with {@link FetchSubPhaseModule#registerFetchSubPhase(Class<? extends FetchSubPhase>)}.
* See {@link FieldDataFieldsFetchSubPhase} for an example.
*/
public class FetchSubPhaseModule extends AbstractModule {

private List<Class<? extends FetchSubPhase>> fetchSubPhases = Lists.newArrayList();

public FetchSubPhaseModule() {
registerFetchSubPhase(ExplainFetchSubPhase.class);
registerFetchSubPhase(FieldDataFieldsFetchSubPhase.class);
registerFetchSubPhase(ScriptFieldsFetchSubPhase.class);
registerFetchSubPhase(FetchSourceSubPhase.class);
registerFetchSubPhase(VersionFetchSubPhase.class);
registerFetchSubPhase(MatchedQueriesFetchSubPhase.class);
registerFetchSubPhase(HighlightPhase.class);
}

public void registerFetchSubPhase(Class<? extends FetchSubPhase> subPhase) {
fetchSubPhases.add(subPhase);
}

@Override
protected void configure() {
Multibinder<FetchSubPhase> parserMapBinder = Multibinder.newSetBinder(binder(), FetchSubPhase.class);
for (Class<? extends FetchSubPhase> clazz : fetchSubPhases) {
parserMapBinder.addBinding().to(clazz);
}
bind(InnerHitsFetchSubPhase.class).asEagerSingleton();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.search.fetch;

import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.internal.SearchContext;

/**
* A parse element for a {@link org.elasticsearch.search.fetch.FetchSubPhase} that is used when parsing a search request.
*/
public abstract class FetchSubPhaseParseElement<SubPhaseContext extends FetchSubPhaseContext> implements SearchParseElement {

@Override
final public void parse(XContentParser parser, SearchContext context) throws Exception {
SubPhaseContext fetchSubPhaseContext = context.getFetchSubPhaseContext(getContextFactory());
// this is to make sure that the SubFetchPhase knows it should execute
fetchSubPhaseContext.setHitExecutionNeeded(true);
innerParse(parser, fetchSubPhaseContext, context);
}

/**
* Implement the actual parsing here.
*/
protected abstract void innerParse(XContentParser parser, SubPhaseContext fetchSubPhaseContext, SearchContext searchContext) throws Exception;

/**
* Return the ContextFactory for this FetchSubPhase.
*/
protected abstract FetchSubPhase.ContextFactory<SubPhaseContext> getContextFactory();
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@
package org.elasticsearch.search.fetch.fielddata;

import com.google.common.collect.Lists;
import org.elasticsearch.search.fetch.FetchSubPhaseContext;

import java.util.List;

/**
* All the required context to pull a field from the field data cache.
*/
public class FieldDataFieldsContext {
public class FieldDataFieldsContext extends FetchSubPhaseContext {

public static class FieldDataField {
private final String name;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.fetch.FetchSubPhaseContext;
import org.elasticsearch.search.internal.InternalSearchHit;
import org.elasticsearch.search.internal.InternalSearchHitField;
import org.elasticsearch.search.internal.SearchContext;
Expand All @@ -43,6 +44,20 @@
*/
public class FieldDataFieldsFetchSubPhase implements FetchSubPhase {

public static final String[] NAMES = {"fielddata_fields", "fielddataFields"};
public static final ContextFactory<FieldDataFieldsContext> CONTEXT_FACTORY = new ContextFactory<FieldDataFieldsContext>() {

@Override
public String getName() {
return NAMES[0];
}

@Override
public FieldDataFieldsContext newContextInstance() {
return new FieldDataFieldsContext();
}
};

@Inject
public FieldDataFieldsFetchSubPhase() {
}
Expand All @@ -66,12 +81,12 @@ public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {

@Override
public boolean hitExecutionNeeded(SearchContext context) {
return context.hasFieldDataFields();
return context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded();
}

@Override
public void hitExecute(SearchContext context, HitContext hitContext) {
for (FieldDataFieldsContext.FieldDataField field : context.fieldDataFields().fields()) {
for (FieldDataFieldsContext.FieldDataField field : context.getFetchSubPhaseContext(CONTEXT_FACTORY).fields()) {
if (hitContext.hit().fieldsOrNull() == null) {
hitContext.hit().fields(new HashMap<String, SearchHitField>(2));
}
Expand Down
Loading

0 comments on commit 26d51c2

Please sign in to comment.