Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Reuse IndexFieldData instances between percolator queries #6845

Closed
wants to merge 2 commits into from
Closed

Reuse IndexFieldData instances between percolator queries #6845

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02afb42
Percolator: Cache IndexFieldData instances when loaded directly to re…
martijnvg Jul 13, 2014
199c849
moved over to the KeyedLock concurrency data structure
martijnvg Jul 16, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
95 changes: 63 additions & 32 deletions src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
View file
Open in desktop
Expand Up @@ -41,7 +41,6 @@
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.indices.fielddata.breaker.NoneCircuitBreakerService;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener;

Expand Down Expand Up @@ -139,6 +138,9 @@ public class IndexFieldDataService extends AbstractIndexComponent {
private final IndicesFieldDataCache indicesFieldDataCache;
private final ConcurrentMap<String, IndexFieldData<?>> loadedFieldData = ConcurrentCollections.newConcurrentMap();
private final KeyedLock.GlobalLockable<String> fieldLoadingLock = new KeyedLock.GlobalLockable<>();
private final ConcurrentMap<String, IndexFieldData<?>> loadedDirectFieldData = ConcurrentCollections.newConcurrentMap();
private final KeyedLock.GlobalLockable<String> fieldDirectLock = new KeyedLock.GlobalLockable<>();

private final Map<String, IndexFieldDataCache> fieldDataCaches = Maps.newHashMap(); // no need for concurrency support, always used under lock

IndexService indexService;
Expand Down Expand Up @@ -193,6 +195,12 @@ public void clear() {
} finally {
fieldLoadingLock.globalLock().unlock();
}
fieldDirectLock.globalLock().lock();
try {
loadedDirectFieldData.clear();
} finally {
fieldDirectLock.globalLock().unlock();
}
}

public void clearField(final String fieldName) {
Expand All @@ -219,6 +227,12 @@ public void clearField(final String fieldName) {
} finally {
fieldLoadingLock.release(fieldName);
}
fieldDirectLock.acquire(fieldName);
try {
loadedDirectFieldData.remove(fieldName);
} finally {
fieldDirectLock.release(fieldName);
}
}

public void onMappingUpdate() {
Expand All @@ -230,6 +244,12 @@ public void onMappingUpdate() {
} finally {
fieldLoadingLock.globalLock().unlock();
}
fieldDirectLock.globalLock().lock();
try {
loadedDirectFieldData.clear();
} finally {
fieldDirectLock.globalLock().unlock();
}
}

@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -301,39 +321,50 @@ public <IFD extends IndexFieldData<?>> IFD getForField(FieldMapper<?> mapper) {

public <IFD extends IndexFieldData<?>> IFD getForFieldDirect(FieldMapper<?> mapper) {
final FieldMapper.Names fieldNames = mapper.names();
final FieldDataType type = mapper.fieldDataType();
if (type == null) {
throw new ElasticsearchIllegalArgumentException("found no fielddata type for field [" + fieldNames.fullName() + "]");
}
final boolean docValues = mapper.hasDocValues();

IndexFieldData.Builder builder = null;
String format = type.getFormat(indexSettings);
if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) {
logger.warn("field [" + fieldNames.fullName() + "] has no doc values, will use default field data format");
format = null;
}
if (format != null) {
builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), format));
if (builder == null) {
logger.warn("failed to find format [" + format + "] for field [" + fieldNames.fullName() + "], will use default");
IndexFieldData<?> fieldData = loadedDirectFieldData.get(fieldNames.indexName());
if (fieldData == null) {
final String key = mapper.names().indexName();
fieldDirectLock.acquire(key);
// By caching IFD instances we prevent that we end up with equal to number of percolator instances
// `loadedFieldData` can't be used b/c we forcefully always use a different IndexFieldDataCache impl
try {
fieldData = loadedDirectFieldData.get(fieldNames.indexName());
if (fieldData == null) {
final FieldDataType type = mapper.fieldDataType();
if (type == null) {
throw new ElasticsearchIllegalArgumentException("found no fielddata type for field [" + fieldNames.fullName() + "]");
}
IndexFieldData.Builder builder = null;
String format = type.getFormat(indexSettings);
final boolean docValues = mapper.hasDocValues();
if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) {
logger.warn("field [" + fieldNames.fullName() + "] has no doc values, will use default field data format");
format = null;
}
if (format != null) {
builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), format));
if (builder == null) {
logger.warn("failed to find format [" + format + "] for field [" + fieldNames.fullName() + "], will use default");
}
}
if (builder == null && docValues) {
builder = docValuesBuildersByType.get(type.getType());
}
if (builder == null) {
builder = buildersByType.get(type.getType());
}
if (builder == null) {
throw new ElasticsearchIllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType());
}
GlobalOrdinalsBuilder globalOrdinalBuilder = new InternalGlobalOrdinalsBuilder(index(), indexSettings);
fieldData = builder.build(index, indexSettings, mapper, new IndexFieldDataCache.None(), circuitBreakerService, indexService.mapperService(), globalOrdinalBuilder);
loadedDirectFieldData.put(fieldNames.indexName(), fieldData);
}
} finally {
fieldDirectLock.release(key);
}
}
if (builder == null && docValues) {
builder = docValuesBuildersByType.get(type.getType());
}
if (builder == null) {
builder = buildersByType.get(type.getType());
}
if (builder == null) {
throw new ElasticsearchIllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType());
}

CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
GlobalOrdinalsBuilder globalOrdinalBuilder = new InternalGlobalOrdinalsBuilder(index(), indexSettings);
@SuppressWarnings("unchecked")
IFD ifd = (IFD) builder.build(index, indexSettings, mapper, new IndexFieldDataCache.None(), circuitBreakerService, indexService.mapperService(), globalOrdinalBuilder);
return ifd;
return (IFD) fieldData;
}

}