Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ML] Transition to typeless (mapping) APIs #39573

Merged
merged 3 commits into from
Mar 4, 2019
Merged

[ML] Transition to typeless (mapping) APIs #39573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
44b7e9b
[ML] Transition to typeless (mapping) APIs (#39256)
davidkyle Feb 28, 2019
2726736
Remove type from DeleteExpiredDataIT
davidkyle Mar 1, 2019
c73cf80
[ML] Switch to typeless APIs in ML native multi-node tests (#39574)
droberts195 Mar 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.io.IOException;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;

public final class MlMetaIndex {
/**
Expand All @@ -21,14 +22,12 @@ public final class MlMetaIndex {
*/
public static final String INDEX_NAME = ".ml-meta";

public static final String TYPE = "doc";

private MlMetaIndex() {}

public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
ElasticsearchMappings.addDefaultMapping(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
Expand Down
5 changes: 3 additions & 2 deletions ...lugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.SortedMap;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;

Expand Down Expand Up @@ -71,7 +72,7 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c

CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, annotationsMapping);
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
Expand Down Expand Up @@ -111,7 +112,7 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
public static XContentBuilder annotationsMapping() throws IOException {
XContentBuilder builder = jsonBuilder()
.startObject()
.startObject(ElasticsearchMappings.DOC_TYPE);
.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Annotation.ANNOTATION.getPreferredName())
Expand Down
105 changes: 59 additions & 46 deletions .../src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.CheckedBiFunction;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
Expand Down Expand Up @@ -62,6 +62,7 @@
import java.util.Map;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;

Expand All @@ -86,8 +87,6 @@
*/
public class ElasticsearchMappings {

public static final String DOC_TYPE = "doc";

/**
* String constants used in mappings
*/
Expand Down Expand Up @@ -137,7 +136,7 @@ private ElasticsearchMappings() {
public static XContentBuilder configMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
Expand Down Expand Up @@ -420,14 +419,14 @@ public static void addMetaInformation(XContentBuilder builder) throws IOExceptio
.endObject();
}

public static XContentBuilder resultsMapping() throws IOException {
return resultsMapping(Collections.emptyList());
public static XContentBuilder resultsMapping(String mappingType) throws IOException {
return resultsMapping(mappingType, Collections.emptyList());
}

public static XContentBuilder resultsMapping(Collection<String> extraTermFields) throws IOException {
public static XContentBuilder resultsMapping(String mappingType, Collection<String> extraTermFields) throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(mappingType);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
Expand Down Expand Up @@ -456,11 +455,12 @@ public static XContentBuilder resultsMapping(Collection<String> extraTermFields)

// end properties
builder.endObject();
// end mapping
// end type
builder.endObject();
// end doc
// end mapping
builder.endObject();


return builder;
}

Expand Down Expand Up @@ -575,18 +575,25 @@ private static void addResultsMapping(XContentBuilder builder) throws IOExceptio
addModelSizeStatsFieldsToMapping(builder);
}

public static XContentBuilder termFieldsMapping(String type, Collection<String> termFields) {
/**
* Generate a keyword mapping for {@code termFields} for the default type
* {@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME}
*
* If the returned mapping is used in index creation and the new index has a matching template
* then the mapping type ({@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME})
* must match the mapping type of the template otherwise the mappings will not be merged correctly.
*
* @param termFields Fields to generate mapping for
* @return The mapping
*/
public static XContentBuilder termFieldsMapping(Collection<String> termFields) {
try {
XContentBuilder builder = jsonBuilder().startObject();
if (type != null) {
builder.startObject(type);
}
builder.startObject(SINGLE_MAPPING_NAME);
builder.startObject(PROPERTIES);
addTermFields(builder, termFields);
builder.endObject();
if (type != null) {
builder.endObject();
}
builder.endObject();
return builder.endObject();
} catch (IOException e) {
throw new RuntimeException(e);
Expand Down Expand Up @@ -872,7 +879,7 @@ private static void addCategoryDefinitionMapping(XContentBuilder builder) throws
public static XContentBuilder stateMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.field(ENABLED, false);
builder.endObject();
Expand Down Expand Up @@ -960,46 +967,47 @@ private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) th
}

public static XContentBuilder auditMessageMapping() throws IOException {
XContentBuilder builder = jsonBuilder().startObject()
.startObject(AuditMessage.TYPE.getPreferredName());
XContentBuilder builder = jsonBuilder().startObject();
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.LEVEL.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.MESSAGE.getPreferredName())
.field(TYPE, TEXT)
.startObject(FIELDS)
.startObject(RAW)
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject()
.startObject(AuditMessage.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(AuditMessage.NODE_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.LEVEL.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.MESSAGE.getPreferredName())
.field(TYPE, TEXT)
.startObject(FIELDS)
.startObject(RAW)
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject();
.endObject()
.startObject(AuditMessage.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(AuditMessage.NODE_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject()
.endObject();

return builder;
}

static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) throws IOException {
List<String> indicesToUpdate = new ArrayList<>();

ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> currentMapping = state.metaData().findMappings(concreteIndices,
new String[] {DOC_TYPE}, MapperPlugin.NOOP_FIELD_FILTER);
new String[0], MapperPlugin.NOOP_FIELD_FILTER);

for (String index : concreteIndices) {
ImmutableOpenMap<String, MappingMetaData> innerMap = currentMapping.get(index);
if (innerMap != null) {
MappingMetaData metaData = innerMap.get(DOC_TYPE);
MappingMetaData metaData = innerMap.valuesIt().next();
try {
@SuppressWarnings("unchecked")
Map<String, Object> meta = (Map<String, Object>) metaData.sourceAsMap().get("_meta");
Expand Down Expand Up @@ -1038,7 +1046,8 @@ static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndic
return indicesToUpdate.toArray(new String[indicesToUpdate.size()]);
}

public static void addDocMappingIfMissing(String alias, CheckedSupplier<XContentBuilder, IOException> mappingSupplier,
public static void addDocMappingIfMissing(String alias,
CheckedBiFunction<String, Collection<String>, XContentBuilder, IOException> mappingSupplier,
Client client, ClusterState state, ActionListener<Boolean> listener) {
AliasOrIndex aliasOrIndex = state.metaData().getAliasAndIndexLookup().get(alias);
if (aliasOrIndex == null) {
Expand All @@ -1058,9 +1067,13 @@ public static void addDocMappingIfMissing(String alias, CheckedSupplier<XContent
}

if (indicesThatRequireAnUpdate.length > 0) {
try (XContentBuilder mapping = mappingSupplier.get()) {
// Use the mapping type of the first index in the update
IndexMetaData indexMetaData = state.metaData().index(indicesThatRequireAnUpdate[0]);
String mappingType = indexMetaData.mapping().type();

try (XContentBuilder mapping = mappingSupplier.apply(mappingType, Collections.emptyList())) {
PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate);
putMappingRequest.type(DOC_TYPE);
putMappingRequest.type(mappingType);
putMappingRequest.source(mapping);
executeAsyncWithOrigin(client, ML_ORIGIN, PutMappingAction.INSTANCE, putMappingRequest,
ActionListener.wrap(response -> {
Expand Down
2 changes: 1 addition & 1 deletion ...plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import java.util.Objects;

public class AuditMessage implements ToXContentObject, Writeable {
public static final ParseField TYPE = new ParseField("audit_message");
private static final ParseField TYPE = new ParseField("audit_message");

public static final ParseField MESSAGE = new ParseField("message");
public static final ParseField LEVEL = new ParseField("level");
Expand Down
11 changes: 7 additions & 4 deletions ...test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;


public class ElasticsearchMappingsTests extends ESTestCase {

Expand Down Expand Up @@ -117,11 +119,12 @@ private void compareFields(Set<String> expected, Set<String> reserved) {
@SuppressWarnings("unchecked")
public void testTermFieldMapping() throws IOException {

XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(null, Arrays.asList("apple", "strawberry",
XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(Arrays.asList("apple", "strawberry",
AnomalyRecord.BUCKET_SPAN.getPreferredName()));

XContentParser parser = createParser(builder);
Map<String, Object> properties = (Map<String, Object>) parser.map().get(ElasticsearchMappings.PROPERTIES);
Map<String, Object> mapping = (Map<String, Object>) parser.map().get(SINGLE_MAPPING_NAME);
Map<String, Object> properties = (Map<String, Object>) mapping.get(ElasticsearchMappings.PROPERTIES);

Map<String, Object> instanceMapping = (Map<String, Object>) properties.get("apple");
assertNotNull(instanceMapping);
Expand Down Expand Up @@ -217,7 +220,7 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object>
}
mapping.put("_meta", meta);

indexMetaData.putMapping(new MappingMetaData(ElasticsearchMappings.DOC_TYPE, mapping));
indexMetaData.putMapping(new MappingMetaData("_doc", mapping));

metaDataBuilder.put(indexMetaData);
}
Expand All @@ -230,7 +233,7 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object>

private Set<String> collectResultsDocFieldNames() throws IOException {
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
return collectFieldNames(ElasticsearchMappings.resultsMapping());
return collectFieldNames(ElasticsearchMappings.resultsMapping("_doc"));
}

private Set<String> collectConfigDocFieldNames() throws IOException {
Expand Down
14 changes: 7 additions & 7 deletions ...lti-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.List;
import java.util.Locale;

import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;

Expand All @@ -35,38 +36,37 @@
public class CategorizationIT extends MlNativeAutodetectIntegTestCase {

private static final String DATA_INDEX = "log-data";
private static final String DATA_TYPE = "log";

private long nowMillis;

@Before
public void setUpData() {
client().admin().indices().prepareCreate(DATA_INDEX)
.addMapping(DATA_TYPE, "time", "type=date,format=epoch_millis",
.addMapping(SINGLE_MAPPING_NAME, "time", "type=date,format=epoch_millis",
"msg", "type=text")
.get();

nowMillis = System.currentTimeMillis();

BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
IndexRequest indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
IndexRequest indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis(),
"msg", "Node 1 started");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis() + 1,
"msg", "Failed to shutdown [error org.aaaa.bbbb.Cccc line 54 caused " +
"by foo exception]");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis(),
"msg", "Node 2 started");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis() + 1,
"msg", "Failed to shutdown [error but this time completely different]");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis, "msg", "Node 3 started");
bulkRequestBuilder.add(indexRequest);

Expand Down
Loading