Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a doc value format to binary fields. #30860

Merged
merged 4 commits into from
Jun 5, 2018
Merged

Add a doc value format to binary fields. #30860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d31392b
Add a doc value format to binary fields.
jpountz May 25, 2018
6b663f1
iter
jpountz May 28, 2018
b0803cc
Merge branch 'master' into binary_doc_value_format
jpountz May 30, 2018
8a68039
Merge branch 'master' into binary_doc_value_format
jpountz Jun 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.DocValueFormat;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.Base64;
Expand Down Expand Up @@ -104,6 +106,10 @@ public String typeName() {
return CONTENT_TYPE;
}

@Override
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
return DocValueFormat.BINARY;
}

@Override
public BytesReference valueForDisplay(Object value) {
Expand Down
45 changes: 45 additions & 0 deletions server/src/main/java/org/elasticsearch/search/DocValueFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Base64;
import java.util.Locale;
import java.util.Objects;
import java.util.function.LongSupplier;
Expand Down Expand Up @@ -121,6 +122,50 @@ public BytesRef parseBytesRef(String value) {
}
};

DocValueFormat BINARY = new DocValueFormat() {

@Override
public String getWriteableName() {
return "binary";
}

@Override
public void writeTo(StreamOutput out) throws IOException {
}

@Override
public Object format(long value) {
throw new UnsupportedOperationException();
}

@Override
public Object format(double value) {
throw new UnsupportedOperationException();
}

@Override
public String format(BytesRef value) {
return Base64.getEncoder()
.withoutPadding()
.encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length));
}

@Override
public long parseLong(String value, boolean roundUp, LongSupplier now) {
throw new UnsupportedOperationException();
}

@Override
public double parseDouble(String value, boolean roundUp, LongSupplier now) {
throw new UnsupportedOperationException();
}

@Override
public BytesRef parseBytesRef(String value) {
return new BytesRef(Base64.getDecoder().decode(value));
}
};

final class DateTime implements DocValueFormat {

public static final String NAME = "date_time";
Expand Down
1 change: 1 addition & 0 deletions server/src/main/java/org/elasticsearch/search/SearchModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,7 @@ private void registerValueFormats() {
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);
}

/**
Expand Down
14 changes: 14 additions & 0 deletions server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public void testSerialization() throws Exception {
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY));
NamedWriteableRegistry registry = new NamedWriteableRegistry(entries);

BytesStreamOutput out = new BytesStreamOutput();
Expand Down Expand Up @@ -82,6 +83,11 @@ public void testSerialization() throws Exception {
out.writeNamedWriteable(DocValueFormat.RAW);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));

out = new BytesStreamOutput();
out.writeNamedWriteable(DocValueFormat.BINARY);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.BINARY, in.readNamedWriteable(DocValueFormat.class));
}

public void testRawFormat() {
Expand All @@ -96,6 +102,14 @@ public void testRawFormat() {
assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc")));
}

public void testBinaryFormat() {
assertEquals("", DocValueFormat.BINARY.format(new BytesRef()));
assertEquals("KmQ", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100})));

assertEquals(new BytesRef(), DocValueFormat.BINARY.parseBytesRef(""));
assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ"));
}

public void testBooleanFormat() {
assertEquals(false, DocValueFormat.BOOLEAN.format(0));
assertEquals(true, DocValueFormat.BOOLEAN.format(1));
Expand Down
14 changes: 11 additions & 3 deletions server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.search.fields;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
Expand Down Expand Up @@ -700,7 +701,7 @@ public void testSingleValueFieldDatatField() throws ExecutionException, Interrup
assertThat(fields.get("test_field").getValue(), equalTo("foobar"));
}

public void testFieldsPulledFromFieldData() throws Exception {
public void testDocValueFields() throws Exception {
createIndex("test");

String mapping = Strings
Expand Down Expand Up @@ -744,6 +745,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
.endObject()
.startObject("binary_field")
.field("type", "binary")
.field("doc_values", true) // off by default on binary fields
.endObject()
.startObject("ip_field")
.field("type", "ip")
Expand All @@ -766,6 +768,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
.field("double_field", 6.0d)
.field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
.field("boolean_field", true)
.field("binary_field", new byte[] {42, 100})
.field("ip_field", "::1")
.endObject()).execute().actionGet();

Expand All @@ -782,6 +785,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
.addDocValueField("double_field")
.addDocValueField("date_field")
.addDocValueField("boolean_field")
.addDocValueField("binary_field")
.addDocValueField("ip_field");
SearchResponse searchResponse = builder.execute().actionGet();

Expand All @@ -790,7 +794,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
Set<String> fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
"ip_field")));
"binary_field", "ip_field")));

assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
Expand All @@ -802,6 +806,8 @@ public void testFieldsPulledFromFieldData() throws Exception {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(),
equalTo(new BytesRef(new byte[] {42, 100})));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));

builder = client().prepareSearch().setQuery(matchAllQuery())
Expand All @@ -815,6 +821,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
.addDocValueField("double_field", "use_field_mapping")
.addDocValueField("date_field", "use_field_mapping")
.addDocValueField("boolean_field", "use_field_mapping")
.addDocValueField("binary_field", "use_field_mapping")
.addDocValueField("ip_field", "use_field_mapping");
searchResponse = builder.execute().actionGet();

Expand All @@ -823,7 +830,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
"ip_field")));
"binary_field", "ip_field")));

assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
Expand All @@ -836,6 +843,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));

builder = client().prepareSearch().setQuery(matchAllQuery())
Expand Down