Skip to content

Commit

Permalink
HSEARCH-2451 Use Elasticsearch "fields" feature to implement faceting
Browse files Browse the repository at this point in the history
  • Loading branch information
@yrodiere @gsmet
yrodiere authored and gsmet committed Nov 23, 2016
1 parent 2e6f0fb commit f82dd62
Show file tree
Hide file tree
Showing 11 changed files with 321 additions and 76 deletions.
27 changes: 26 additions & 1 deletion ...earch/src/main/java/org/hibernate/search/elasticsearch/impl/ElasticsearchHSQueryImpl.java
View file
Expand Up @@ -50,6 +50,8 @@
import org.hibernate.search.engine.integration.impl.ExtendedSearchIntegrator;
import org.hibernate.search.engine.metadata.impl.BridgeDefinedField;
import org.hibernate.search.engine.metadata.impl.DocumentFieldMetadata;
import org.hibernate.search.engine.metadata.impl.FacetMetadata;
import org.hibernate.search.engine.metadata.impl.TypeMetadata;
import org.hibernate.search.engine.service.spi.ServiceReference;
import org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity;
import org.hibernate.search.engine.spi.EntityIndexBinding;
Expand Down Expand Up @@ -290,6 +292,7 @@ private void execute() {
private class IndexSearcher {

private final Map<String, Class<?>> entityTypesByName = new HashMap<>();
private final Map<String, Class<?>> documentBuilderIndexedEntityByName = new HashMap<>();
private final Set<String> indexNames = new HashSet<>();
private final String completeQueryAsString;

Expand Down Expand Up @@ -328,7 +331,7 @@ private IndexSearcher() {
JsonBuilder.Object facets = JsonBuilder.object();

for ( Entry<String, FacetingRequest> facetRequestEntry : getFacetManager().getFacetRequests().entrySet() ) {
ToElasticsearch.addFacetingRequest( facets, facetRequestEntry.getValue() );
addFacetingRequest( facets, facetRequestEntry.getValue() );
}

completeQuery.add( "aggregations", facets );
Expand Down Expand Up @@ -431,6 +434,28 @@ private Iterable<Class<?>> getQueriedEntityTypes() {
}
}

private void addFacetingRequest(JsonBuilder.Object facets, FacetingRequest facetingRequest) {
String facetFieldAbsoluteName = facetingRequest.getFieldName();
FacetMetadata facetMetadata = null;
for ( Class<?> entityType : getQueriedEntityTypes() ) {
EntityIndexBinding binding = extendedIntegrator.getIndexBinding( entityType );
TypeMetadata typeMetadata = binding.getDocumentBuilder().getTypeMetadata();
facetMetadata = typeMetadata.getFacetMetadataFor( facetFieldAbsoluteName );
if ( facetMetadata != null ) {
break;
}
}

if ( facetMetadata == null ) {
throw LOG.unknownFieldNameForFaceting( facetingRequest.getFacetingName(), facetingRequest.getFieldName() );
}

String sourceFieldAbsoluteName = facetMetadata.getSourceField().getAbsoluteName();
String facetSubfieldName = facetMetadata.getPath().getRelativeName();

ToElasticsearch.addFacetingRequest( facets, facetingRequest, sourceFieldAbsoluteName, facetSubfieldName );
}

private Sort getSort(SortField sortField) {
if ( sortField instanceof DistanceSortField ) {
DistanceSortField distanceSortField = (DistanceSortField) sortField;
Expand Down
60 changes: 19 additions & 41 deletions .../src/main/java/org/hibernate/search/elasticsearch/impl/ElasticsearchIndexWorkVisitor.java
View file
Expand Up @@ -11,9 +11,6 @@
import java.util.Set;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexableField;
Expand Down Expand Up @@ -244,9 +241,25 @@ private JsonObject convertToJson(Document document, Class<?> entityType) {
throw new AssertionFailure( "Unexpected container type: " + containerType );
}
}
else if ( !fieldPath.equals( ProjectionConstants.OBJECT_CLASS ) &&
!FacetsConfig.DEFAULT_INDEX_FIELD_NAME.equals( fieldPath ) &&
!isDocValueField( field ) ) {
else if ( FacetsConfig.DEFAULT_INDEX_FIELD_NAME.equals( field.name() ) ) {
/*
* Lucene-specific fields for facets.
* Just ignore such fields: Elasticsearch handles that internally.
*/
continue;
}
else if ( isDocValueField( field ) ) {
/*
* Doc value fields for facets or sorts.
* Just ignore such fields: Elasticsearch handles that internally.
*/
continue;
}
else if ( fieldPath.equals( ProjectionConstants.OBJECT_CLASS ) ) {
// Object class: no need to index that in Elasticsearch, because documents are typed.
continue;
}
else {
DocumentFieldMetadata documentFieldMetadata = indexBinding.getDocumentBuilder().getTypeMetadata()
.getDocumentFieldMetadataFor( field.name() );

Expand Down Expand Up @@ -318,41 +331,6 @@ else if ( numericValue != null ) {
}
}
}
else if ( FacetsConfig.DEFAULT_INDEX_FIELD_NAME.equals( field.name() )
&& field instanceof SortedSetDocValuesField ) {
// String facet fields
String[] facetParts = FacetsConfig.stringToPath( field.binaryValue().utf8ToString() );
if ( facetParts == null || facetParts.length != 2 ) {
continue;
}
String facetFieldPath = facetParts[0];
String value = facetParts[1];

// if it's not just a facet field, we ignore it as the field is going to be created by the standard
// mechanism
if ( indexBinding.getDocumentBuilder().getTypeMetadata().getDocumentFieldMetadataFor( facetFieldPath ) != null ) {
continue;
}

accessorBuilder.buildForPath( facetFieldPath ).add( root, value != null ? new JsonPrimitive( value ) : null );
}
else if ( isDocValueField( field ) && field instanceof NumericDocValuesField ) {
// Numeric facet fields: we also get fields created for sorting so we need to exclude them.
if ( indexBinding.getDocumentBuilder().getTypeMetadata().getDocumentFieldMetadataFor( field.name() ) != null ) {
continue;
}

Number value;
if ( field instanceof DoubleDocValuesField ) {
// double values are encoded so we need to decode them
value = Double.longBitsToDouble( field.numericValue().longValue() );
}
else {
value = field.numericValue();
}

accessorBuilder.buildForPath( fieldPath ).add( root, value != null ? new JsonPrimitive( value ) : null );
}
}

return root;
Expand Down
23 changes: 15 additions & 8 deletions elasticsearch/src/main/java/org/hibernate/search/elasticsearch/impl/ToElasticsearch.java
View file
Expand Up @@ -50,6 +50,11 @@
*/
public class ToElasticsearch {

/*
* A specific suffix for facet fields that avoids conflicts with existing names.
*/
public static final String FACET_FIELD_SUFFIX = "__HSearch_Facet";

private static final Log LOG = LoggerFactory.make( Log.class );

private static final int DEFAULT_SLOP = 0;
Expand All @@ -59,22 +64,24 @@ public class ToElasticsearch {
private ToElasticsearch() {
}

public static void addFacetingRequest(JsonBuilder.Object jsonQuery, FacetingRequest facetingRequest) {
String fieldName = facetingRequest.getFieldName();
public static void addFacetingRequest(JsonBuilder.Object jsonQuery, FacetingRequest facetingRequest,
String sourceFieldAbsoluteName, String facetRelativeName) {
String aggregationFieldName = sourceFieldAbsoluteName + "." + facetRelativeName + FACET_FIELD_SUFFIX;

if ( facetingRequest instanceof DiscreteFacetRequest ) {
JsonObject termsJsonQuery = JsonBuilder.object().add( "terms",
JsonBuilder.object()
.addProperty( "field", fieldName )
.addProperty( "field", aggregationFieldName )
.addProperty( "size", facetingRequest.getMaxNumberOfFacets() == -1 ? Integer.MAX_VALUE : facetingRequest.getMaxNumberOfFacets() )
.add( "order", fromFacetSortOrder( facetingRequest.getSort() ) )
.addProperty( "min_doc_count", facetingRequest.hasZeroCountsIncluded() ? 0 : 1 )
).build();

if ( isNested( fieldName ) ) {
if ( isNested( sourceFieldAbsoluteName ) ) {
JsonBuilder.Object facetJsonQuery = JsonBuilder.object();
facetJsonQuery.add( "nested", JsonBuilder.object()
.addProperty( "path", FieldHelper.getEmbeddedFieldPath( fieldName ) ) );
facetJsonQuery.add( "aggregations", JsonBuilder.object().add( facetingRequest.getFacetingName(), termsJsonQuery ) );
.addProperty( "path", FieldHelper.getEmbeddedFieldPath( sourceFieldAbsoluteName ) + "." + facetRelativeName + FACET_FIELD_SUFFIX ) );
facetJsonQuery.add( "aggregations", JsonBuilder.object().add( aggregationFieldName, termsJsonQuery ) );
jsonQuery.add( facetingRequest.getFacetingName(), facetJsonQuery );
}
else {
Expand All @@ -92,9 +99,9 @@ else if ( facetingRequest instanceof RangeFacetRequest<?> ) {
comparisonFragment.addProperty( facetRange.isMaxIncluded() ? "lte" : "lt", facetRange.getMax() );
}

JsonObject rangeQuery = wrapQueryForNestedIfRequired( fieldName,
JsonObject rangeQuery = wrapQueryForNestedIfRequired( aggregationFieldName,
JsonBuilder.object().add( "range",
JsonBuilder.object().add( fieldName, comparisonFragment ) ).build() );
JsonBuilder.object().add( aggregationFieldName, comparisonFragment ) ).build() );

jsonQuery.add( facetingRequest.getFacetingName() + "-" + facetRange.getIdentifier(),
JsonBuilder.object().add( "filter", rangeQuery ) );
Expand Down
37 changes: 19 additions & 18 deletions .../org/hibernate/search/elasticsearch/schema/impl/DefaultElasticsearchSchemaTranslator.java
View file
Expand Up @@ -16,6 +16,7 @@
import org.hibernate.search.analyzer.impl.AnalyzerReference;
import org.hibernate.search.annotations.Store;
import org.hibernate.search.elasticsearch.impl.GsonService;
import org.hibernate.search.elasticsearch.impl.ToElasticsearch;
import org.hibernate.search.elasticsearch.logging.impl.Log;
import org.hibernate.search.elasticsearch.schema.impl.model.DataType;
import org.hibernate.search.elasticsearch.schema.impl.model.DynamicType;
Expand Down Expand Up @@ -148,16 +149,12 @@ private void addPropertyMapping(ElasticsearchMappingBuilder mappingBuilder, Docu

addNullValue( propertyMapping, fieldMetadata );

// Create facet fields if needed: if the facet has the same name as the field, we don't need to create an
// extra field for it
for ( FacetMetadata facetMetadata : fieldMetadata.getFacetMetadata() ) {
if ( !facetMetadata.getAbsoluteName().equals( fieldMetadata.getAbsoluteName() ) ) {
try {
addPropertyMapping( mappingBuilder, facetMetadata );
}
catch (IncompleteDataException e) {
LOG.debug( "Not adding a mapping for facet " + facetMetadata.getAbsoluteName() + " because of incomplete data", e );
}
try {
addSubfieldMapping( propertyMapping, mappingBuilder, facetMetadata );
}
catch (IncompleteDataException e) {
LOG.debug( "Not adding a mapping for facet " + facetMetadata.getAbsoluteName() + " because of incomplete data", e );
}
}

Expand Down Expand Up @@ -214,17 +211,21 @@ else if ( SpatialHelper.isSpatialFieldLatitude( propertyPath ) ) {
}
}

private void addPropertyMapping(ElasticsearchMappingBuilder mappingBuilder, FacetMetadata facetMetadata) {
String propertyPath = facetMetadata.getAbsoluteName();
/*
* Adds an Elasticsearch "field" to an existing property for a facet.
* <p>Note that "field" in ES has a very specific meaning, which is not the meaning it has in Lucene or Hibernate Search.
*/
private void addSubfieldMapping(PropertyMapping propertyMapping, ElasticsearchMappingBuilder mappingBuilder, FacetMetadata facetMetadata) {
String facetFieldName = facetMetadata.getPath().getRelativeName() + ToElasticsearch.FACET_FIELD_SUFFIX;

PropertyMapping propertyMapping = new PropertyMapping();
PropertyMapping fieldMapping = new PropertyMapping();

addTypeOptions( propertyMapping, facetMetadata );
propertyMapping.setStore( false );
propertyMapping.setIndex( IndexType.NOT_ANALYZED );
addTypeOptions( fieldMapping, facetMetadata );
fieldMapping.setStore( false );
fieldMapping.setIndex( IndexType.NOT_ANALYZED );

// Do this last, when we're sure no exception will be thrown for this mapping
mappingBuilder.setPropertyAbsolute( propertyPath, propertyMapping );
propertyMapping.addField( facetFieldName, fieldMapping );
}

/**
Expand Down Expand Up @@ -279,7 +280,7 @@ private void addTypeOptions(PropertyMapping propertyMapping, BridgeDefinedField
addTypeOptions( bridgeDefinedField.getAbsoluteName(), propertyMapping, type );
}

private void addTypeOptions(PropertyMapping propertyMapping, FacetMetadata facetMetadata) {
private void addTypeOptions(PropertyMapping fieldMapping, FacetMetadata facetMetadata) {
ExtendedFieldType type;

switch ( facetMetadata.getEncoding() ) {
Expand All @@ -303,7 +304,7 @@ private void addTypeOptions(PropertyMapping propertyMapping, FacetMetadata facet
}
}

addTypeOptions( facetMetadata.getAbsoluteName(), propertyMapping, type );
addTypeOptions( facetMetadata.getAbsoluteName(), fieldMapping, type );
}

private DataType addTypeOptions(String fieldName, PropertyMapping propertyMapping, ExtendedFieldType extendedType) {
Expand Down
37 changes: 36 additions & 1 deletion ...a/org/hibernate/search/elasticsearch/schema/impl/DefaultElasticsearchSchemaValidator.java
View file
Expand Up @@ -103,7 +103,10 @@ public void validate(IndexMetadata expectedIndexMetadata, ExecutionOptions execu
}

private Object formatIntro(ValidationContext context) {
if ( StringHelper.isNotEmpty( context.getPath() ) ) {
if ( StringHelper.isNotEmpty( context.getFieldName() ) ) {
return MESSAGES.errorIntro( context.getIndexName(), context.getMappingName(), context.getPath(), context.getFieldName() );
}
else if ( StringHelper.isNotEmpty( context.getPath() ) ) {
return MESSAGES.errorIntro( context.getIndexName(), context.getMappingName(), context.getPath() );
}
else if ( StringHelper.isNotEmpty( context.getMappingName() ) ) {
Expand Down Expand Up @@ -325,7 +328,39 @@ private void validatePropertyMapping(ValidationErrorCollector errorCollector,
expectedMapping.getNullValue(), actualMapping.getNullValue() );

validateEqualWithDefault( errorCollector, "analyzer", expectedMapping.getAnalyzer(), actualMapping.getAnalyzer(), null );

validateTypeMapping( errorCollector, expectedMapping, actualMapping );

validatePropertyMappingFields( errorCollector, expectedMapping, actualMapping );
}

private void validatePropertyMappingFields(ValidationErrorCollector errorCollector,
PropertyMapping expectedMapping, PropertyMapping actualMapping) {
// Unknown fields are ignored, we only validate expected fields
Map<String, PropertyMapping> expectedFieldMappings = expectedMapping.getFields();
Map<String, PropertyMapping> actualFieldMappings = actualMapping.getFields();
if ( expectedFieldMappings != null ) {
for ( Map.Entry<String, PropertyMapping> entry : expectedFieldMappings.entrySet() ) {
String fieldName = entry.getKey();
PropertyMapping expectedFieldMapping = entry.getValue();

PropertyMapping actualFieldMapping = actualFieldMappings == null ?
null : actualFieldMappings.get( fieldName );

errorCollector.setFieldName( fieldName );
try {
if ( actualFieldMapping == null ) {
errorCollector.addError( MESSAGES.propertyFieldMissing() );
continue;
}
// Validate with the same method as properties, since the content is about the same
validatePropertyMapping( errorCollector, expectedFieldMapping, actualFieldMapping );
}
catch (ElasticsearchSchemaValidationException e) {
errorCollector.setFieldName( null );
}
}
}
}

private static void validateJsonPrimitive(ValidationErrorCollector errorCollector,
Expand Down
10 changes: 10 additions & 0 deletions .../java/org/hibernate/search/elasticsearch/schema/impl/ElasticsearchValidationMessages.java
View file
Expand Up @@ -34,6 +34,11 @@ public interface ElasticsearchValidationMessages {
)
String errorIntro(String indexName, String mappingName, String path);

@Message(
value = "Index '%1$s', mapping '%2$s', property '%3$s', field '%4$s':"
)
String errorIntro(String indexName, String mappingName, String path, String field);

@Message(
value = "Missing type mapping"
)
Expand All @@ -44,6 +49,11 @@ public interface ElasticsearchValidationMessages {
)
String propertyMissing();

@Message(
value = "Missing field mapping"
)
String propertyFieldMissing();

@Message(
value = "Invalid value for attribute '%1$s'. Expected '%2$s', actual is '%3$s'"
)
Expand Down
12 changes: 10 additions & 2 deletions ...earch/src/main/java/org/hibernate/search/elasticsearch/schema/impl/ValidationContext.java
View file
Expand Up @@ -12,12 +12,14 @@ final class ValidationContext {
private final String indexName;
private final String mappingName;
private final String path;
private final String fieldName;

public ValidationContext(String indexName, String mappingName, String path) {
public ValidationContext(String indexName, String mappingName, String path, String fieldName) {
super();
this.indexName = indexName;
this.mappingName = mappingName;
this.path = path;
this.fieldName = fieldName;
}

public String getIndexName() {
Expand All @@ -32,13 +34,18 @@ public String getPath() {
return path;
}

public String getFieldName() {
return fieldName;
}

@Override
public boolean equals(Object obj) {
if ( obj != null && getClass().equals( obj.getClass() ) ) {
ValidationContext other = (ValidationContext) obj;
return Objects.equals( indexName, other.indexName )
&& Objects.equals( mappingName, other.mappingName )
&& Objects.equals( path, other.path );
&& Objects.equals( path, other.path )
&& Objects.equals( fieldName, other.fieldName );
}
return false;
}
Expand All @@ -50,6 +57,7 @@ public int hashCode() {
result = prime * result + Objects.hashCode( indexName );
result = prime * result + Objects.hashCode( mappingName );
result = prime * result + Objects.hashCode( path );
result = prime * result + Objects.hashCode( fieldName );
return result;
}
}

0 comments on commit f82dd62

Please sign in to comment.