From d376d9d3ddc46423d42561b56852b6a3dd6f21e4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 12 Nov 2018 11:00:50 +0100 Subject: [PATCH] Make typeless APIs usable with indices whose type name is different from `_doc`. This commit makes `document`, `update`, `explain`, `termvectors` and `mapping` typeless APIs work on indices that have a type whose name is not `_doc`. Unfortunately, this needs to be a bit of a hack since I didn't want calls with random type names to see documents with the type name that the user had chosen upon type creation. The `explain` and `termvectors` do not support being called without a type for now so the test is just using `_doc` as a type for now, we will need to fix tests later but this shouldn't require further changes server-side since passing `_doc` as a type name is what typeless APIs do internally anyway. Relates #35190 --- .../test/delete/70_mix_typeless_typeful.yml | 40 ++++++++++++ .../test/explain/40_mix_typeless_typeful.yml | 56 +++++++++++++++++ .../test/get/100_mix_typeless_typeful.yml | 46 ++++++++++++++ .../test/index/70_mix_typeless_typeful.yml | 62 +++++++++++++++++++ .../70_mix_typeless_typeful.yml | 23 +++++++ .../20_mix_typeless_typeful.yml | 43 +++++++++++++ .../termvectors/50_mix_typeless_typeful.yml | 45 ++++++++++++++ .../test/update/90_mix_typeless_typeful.yml | 39 ++++++++++++ .../explain/TransportExplainAction.java | 8 +-- .../metadata/MetaDataMappingService.java | 14 ++++- .../index/get/ShardGetService.java | 16 ++--- .../index/mapper/DocumentParser.java | 3 +- .../index/mapper/MapperService.java | 8 --- .../elasticsearch/index/shard/IndexShard.java | 43 +++++++++---- .../index/termvectors/TermVectorsService.java | 8 +-- .../search/DefaultSearchContext.java | 5 ++ .../index/mapper/DocumentParserTests.java | 17 +++++ .../index/shard/IndexShardTests.java | 62 ++++++++++++++++++- .../index/shard/ShardGetServiceTests.java | 27 ++++++++ 19 files changed, 522 insertions(+), 43 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/delete/70_mix_typeless_typeful.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/explain/40_mix_typeless_typeful.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get/100_mix_typeless_typeful.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/70_mix_typeless_typeful.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/70_mix_typeless_typeful.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/50_mix_typeless_typeful.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/90_mix_typeless_typeful.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/70_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/70_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..df7ee544cd219 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/70_mix_typeless_typeful.yml @@ -0,0 +1,40 @@ +--- +"DELETE with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: Typeless APIs were introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "keyword" + + - do: + index: + index: index + type: not_doc + id: 1 + body: { foo: bar } + + - do: + catch: bad_request + delete: + index: index + type: some_random_type + id: 1 + + - do: + delete: + index: index + id: 1 + + - match: { _index: "index" } + - match: { _type: "_doc" } + - match: { _id: "1"} + - match: { _version: 2} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/40_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/40_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..baefba7c312de --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/40_mix_typeless_typeful.yml @@ -0,0 +1,56 @@ +--- +"Explain with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: Typeless APIs were introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "keyword" + + - do: + index: + index: index + type: not_doc + id: 1 + body: { foo: bar } + + - do: + indices.refresh: {} + + - do: + catch: missing + explain: + index: index + type: some_random_type + id: 1 + body: + query: + match_all: {} + + - match: { _index: "index" } + - match: { _type: "some_random_type" } + - match: { _id: "1"} + - match: { matched: false} + + - do: + explain: + index: index + type: _doc #todo: make _explain typeless and remove this + id: 1 + body: + query: + match_all: {} + + - match: { _index: "index" } + - match: { _type: "_doc" } + - match: { _id: "1"} + - is_true: matched + - match: { explanation.value: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/100_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/100_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..71907461da3ea --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/100_mix_typeless_typeful.yml @@ -0,0 +1,46 @@ +--- +"GET with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: Typeless APIs were introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "keyword" + + - do: + index: + index: index + type: not_doc + id: 1 + body: { foo: bar } + + - do: + catch: missing + get: + index: index + type: some_random_type + id: 1 + + - match: { _index: "index" } + - match: { _type: "some_random_type" } + - match: { _id: "1"} + - match: { found: false} + + - do: + get: + index: index + id: 1 + + - match: { _index: "index" } + - match: { _type: "_doc" } + - match: { _id: "1"} + - match: { _version: 1} + - match: { _source: { foo: bar }} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..5e225ec1ad30a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_mix_typeless_typeful.yml @@ -0,0 +1,62 @@ +--- +"Index with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: Typeless APIs were introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "keyword" + + - do: + index: + index: index + id: 1 + body: { foo: bar } + + - match: { _index: "index" } + - match: { _type: "_doc" } + - match: { _id: "1"} + - match: { _version: 1} + + - do: + get: # not using typeless API on purpose + index: index + type: not_doc + id: 1 + + - match: { _index: "index" } + - match: { _type: "not_doc" } # the important bit to check + - match: { _id: "1"} + - match: { _version: 1} + - match: { _source: { foo: bar }} + + + - do: + index: + index: index + body: { foo: bar } + + - match: { _index: "index" } + - match: { _type: "_doc" } + - match: { _version: 1} + - set: { _id: id } + + - do: + get: # using typeful API on purpose + index: index + type: not_doc + id: '$id' + + - match: { _index: "index" } + - match: { _type: "not_doc" } # the important bit to check + - match: { _id: $id} + - match: { _version: 1} + - match: { _source: { foo: bar }} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/70_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/70_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..89e0d42a9e799 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/70_mix_typeless_typeful.yml @@ -0,0 +1,23 @@ +--- +"GET mapping with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: include_type_name was introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "keyword" + + - do: + indices.get_mapping: + include_type_name: false + index: index + + - match: { index.mappings.properties.foo.type: "keyword" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..02b5a5f920062 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml @@ -0,0 +1,43 @@ +--- +"PUT mapping with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: include_type_name was introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "keyword" + + - do: + indices.put_mapping: + include_type_name: false + index: index + body: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: false + index: index + + - match: { index.mappings.properties.foo.type: "keyword" } + - match: { index.mappings.properties.bar.type: "long" } + + - do: + catch: bad_request + indices.put_mapping: + index: index + body: + some_other_type: + properties: + bar: + type: "long" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/50_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/50_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..403f2b5b8cf67 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/50_mix_typeless_typeful.yml @@ -0,0 +1,45 @@ +--- +"Term vectors with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: Typeless APIs were introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "text" + term_vector: "with_positions" + + - do: + index: + index: index + type: not_doc + id: 1 + body: { foo: bar } + + - do: + indices.refresh: {} + + - do: + termvectors: + index: index + type: _doc # todo: remove when termvectors support typeless API + id: 1 + + - is_true: found + - match: {_type: _doc} + - match: {term_vectors.foo.terms.bar.term_freq: 1} + + - do: + termvectors: + index: index + type: some_random_type + id: 1 + + - is_false: found diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/90_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/90_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..066f0989c35b2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/90_mix_typeless_typeful.yml @@ -0,0 +1,39 @@ +--- +"Update with typeless API on an index that has types": + + - skip: + version: " - 6.99.99" + reason: Typeless APIs were introduced in 7.0.0 + + - do: + indices.create: # not using include_type_name: false on purpose + index: index + body: + mappings: + not_doc: + properties: + foo: + type: "keyword" + + - do: + index: + index: index + type: not_doc + id: 1 + body: { foo: bar } + + - do: + update: + index: index + id: 1 + body: + doc: + foo: baz + + - do: + get: + index: index + type: not_doc + id: 1 + + - match: { _source.foo: baz } diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index 85fe196933bff..b04b34fb3c3d3 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -35,6 +35,8 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchService; @@ -109,10 +111,8 @@ protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId SearchContext context = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT); Engine.GetResult result = null; try { - Term uidTerm = context.mapperService().createUidTerm(request.type(), request.id()); - if (uidTerm == null) { - return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), false); - } + // No need to check the type, IndexShard#get does it for is + Term uidTerm = new Term(IdFieldMapper.NAME, Uid.encodeId(request.id())); result = context.indexShard().get(new Engine.Get(false, false, request.type(), request.id(), uidTerm)); if (!result.exists()) { return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), false); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 1832d73524161..112f76f79d3a8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -263,7 +263,7 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt updateList.add(indexMetaData); // try and parse it (no need to add it here) so we can bail early in case of parsing exception DocumentMapper newMapper; - DocumentMapper existingMapper = mapperService.documentMapper(request.type()); + DocumentMapper existingMapper = mapperService.documentMapper(); if (MapperService.DEFAULT_MAPPING.equals(request.type())) { // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default newMapper = mapperService.parse(request.type(), mappingUpdateSource, false); @@ -295,12 +295,22 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt // we use the exact same indexService and metadata we used to validate above here to actually apply the update final Index index = indexMetaData.getIndex(); final MapperService mapperService = indexMapperServices.get(index); + String typeForUpdate = mappingType; // the type to use to apply the mapping update + if (MapperService.SINGLE_MAPPING_NAME.equals(typeForUpdate)) { + // If the user gave _doc as a special type value or if (s)he is using the new typeless APIs, + // then we apply the mapping update to the existing type. This allows to move to typeless + // APIs with indices whose type name is different from `_doc`. + DocumentMapper mapper = mapperService.documentMapper(); + if (mapper != null) { + typeForUpdate = mapper.type(); + } + } CompressedXContent existingSource = null; DocumentMapper existingMapper = mapperService.documentMapper(mappingType); if (existingMapper != null) { existingSource = existingMapper.mappingSource(); } - DocumentMapper mergedMapper = mapperService.merge(mappingType, mappingUpdateSource, MergeReason.MAPPING_UPDATE); + DocumentMapper mergedMapper = mapperService.merge(typeForUpdate, mappingUpdateSource, MergeReason.MAPPING_UPDATE); CompressedXContent updatedSource = mergedMapper.mappingSource(); if (existingSource != null) { diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 3b84b00b0f8c3..fc1796dfcc523 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -39,10 +39,12 @@ import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; import org.elasticsearch.index.fieldvisitor.FieldsVisitor; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -157,13 +159,11 @@ private GetResult innerGet(String type, String id, String[] gFields, boolean rea Engine.GetResult get = null; if (type != null) { - Term uidTerm = mapperService.createUidTerm(type, id); - if (uidTerm != null) { - get = indexShard.get(new Engine.Get(realtime, readFromTranslog, type, id, uidTerm) - .version(version).versionType(versionType)); - if (get.exists() == false) { - get.close(); - } + Term uidTerm = new Term(IdFieldMapper.NAME, Uid.encodeId(id)); + get = indexShard.get(new Engine.Get(realtime, readFromTranslog, type, id, uidTerm) + .version(version).versionType(versionType)); + if (get.exists() == false) { + get.close(); } } @@ -202,7 +202,7 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[] } } - DocumentMapper docMapper = mapperService.documentMapper(type); + DocumentMapper docMapper = mapperService.documentMapper(); if (gFields != null && gFields.length > 0) { for (String field : gFields) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 82baef2780943..cccb221d69e9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -106,7 +106,8 @@ private void validateType(SourceToParse source) { throw new IllegalArgumentException("It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]"); } - if (Objects.equals(source.type(), docMapper.type()) == false) { + if (Objects.equals(source.type(), docMapper.type()) == false && + MapperService.SINGLE_MAPPING_NAME.equals(source.type()) == false) { // used by typeless APIs throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 076f6e7ebe030..4b769e488fedf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -26,7 +26,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; -import org.apache.lucene.index.Term; import org.elasticsearch.Assertions; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; @@ -772,11 +771,4 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { } } - /** Return a term that uniquely identifies the document, or {@code null} if the type is not allowed. */ - public Term createUidTerm(String type, String id) { - if (mapper == null || mapper.type().equals(type) == false) { - return null; - } - return new Term(IdFieldMapper.NAME, Uid.encodeId(id)); - } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 0638ce32a147c..b6be5f4a9b267 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -31,7 +31,6 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.Assertions; import org.elasticsearch.ElasticsearchException; @@ -76,6 +75,7 @@ import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.Engine.GetResult; import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.engine.EngineFactory; @@ -814,23 +814,23 @@ private Engine.DeleteResult applyDeleteOperation(long seqNo, long opPrimaryTerm, } catch (MapperParsingException | IllegalArgumentException | TypeMissingException e) { return new Engine.DeleteResult(e, version, operationPrimaryTerm, seqNo, false); } - final Term uid = extractUidForDelete(type, id); + if (resolveType(type).equals(mapperService.documentMapper().type()) == false) { + // We should never get there due to the fact that we generate mapping updates on deletes, + // but we still prefer to have a hard exception here as we would otherwise delete a + // document in the wrong type. + throw new IllegalStateException("Deleting document from type [" + resolveType(type) + "] while current type is [" + + mapperService.documentMapper().type() + "]"); + } + final Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(id)); final Engine.Delete delete = prepareDelete(type, id, uid, seqNo, opPrimaryTerm, version, versionType, origin); return delete(getEngine(), delete); } - private static Engine.Delete prepareDelete(String type, String id, Term uid, long seqNo, long primaryTerm, long version, + private Engine.Delete prepareDelete(String type, String id, Term uid, long seqNo, long primaryTerm, long version, VersionType versionType, Engine.Operation.Origin origin) { long startTime = System.nanoTime(); - return new Engine.Delete(type, id, uid, seqNo, primaryTerm, version, versionType, origin, startTime); - } - - private Term extractUidForDelete(String type, String id) { - // This is only correct because we create types dynamically on delete operations - // otherwise this could match the same _id from a different type - BytesRef idBytes = Uid.encodeId(id); - return new Term(IdFieldMapper.NAME, idBytes); + return new Engine.Delete(resolveType(type), id, uid, seqNo, primaryTerm, version, versionType, origin, startTime); } private Engine.DeleteResult delete(Engine engine, Engine.Delete delete) throws IOException { @@ -852,6 +852,10 @@ private Engine.DeleteResult delete(Engine engine, Engine.Delete delete) throws I public Engine.GetResult get(Engine.Get get) { readAllowed(); + DocumentMapper mapper = mapperService.documentMapper(); + if (mapper == null || mapper.type().equals(resolveType(get.type())) == false) { + return GetResult.NOT_EXISTS; + } return getEngine().get(get, this::acquireSearcher); } @@ -2270,8 +2274,23 @@ private static void persistMetadata( } } + /** + * If an index/update/get/delete operation is using the special `_doc` type, then we replace + * it with the actual type that is being used in the mappings so that users may use typeless + * APIs with indices that have types. + */ + private String resolveType(String type) { + if (MapperService.SINGLE_MAPPING_NAME.equals(type)) { + DocumentMapper docMapper = mapperService.documentMapper(); + if (docMapper != null) { + return docMapper.type(); + } + } + return type; + } + private DocumentMapperForType docMapper(String type) { - return mapperService.documentMapperWithAutoCreate(type); + return mapperService.documentMapperWithAutoCreate(resolveType(type)); } private EngineConfig newEngineConfig() { diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index bbc7c755a67e9..68f175f7ed6ae 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.DocumentMapperForType; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -50,6 +51,7 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.StringFieldType; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.search.dfs.AggregatedDfs; @@ -82,11 +84,7 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ final long startTime = nanoTimeSupplier.getAsLong(); final TermVectorsResponse termVectorsResponse = new TermVectorsResponse(indexShard.shardId().getIndex().getName(), request.type(), request.id()); - final Term uidTerm = indexShard.mapperService().createUidTerm(request.type(), request.id()); - if (uidTerm == null) { - termVectorsResponse.setExists(false); - return termVectorsResponse; - } + final Term uidTerm = new Term(IdFieldMapper.NAME, Uid.encodeId(request.id())); Fields termVectorsByField = null; AggregatedDfs dfs = null; diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 0b17ec72fbb17..7808b4ed51de5 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -302,6 +302,11 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query) private Query createTypeFilter(String[] types) { if (types != null && types.length >= 1) { + if (Arrays.asList(types).contains(MapperService.SINGLE_MAPPING_NAME)) { + // Typeless APIs use _doc as a type name internally, so we need + // to make filtering on _type:_doc a no-op. + return null; + } MappedFieldType ft = mapperService().fullName(TypeFieldMapper.NAME); if (ft != null) { // ft might be null if no documents have been indexed yet diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index b3bdd9f33cf18..2ec49e5b20431 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -1550,4 +1550,21 @@ public void testDynamicDottedFieldNameWithFieldAlias() throws Exception { assertEquals("Could not dynamically add mapping for field [alias-field.dynamic-field]. " + "Existing mapping for [alias-field] must be of type object but found [alias].", exception.getMessage()); } + + public void testTypeless() throws IOException { + DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject().startObject("type").startObject("properties") + .startObject("foo").field("type", "keyword").endObject() + .endObject().endObject().endObject()); + DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping)); + + BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder() + .startObject() + .field("foo", "1234") + .endObject()); + + ParsedDocument doc = mapper.parse(SourceToParse.source("test", "_doc", "1", bytes, XContentType.JSON)); + assertNull(doc.dynamicMappingsUpdate()); // no update since we reused the existing type + } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 8eede7542bd75..b571a8399e3b9 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -65,6 +65,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -77,8 +78,10 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.Engine.DeleteResult; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.engine.InternalEngine; @@ -1326,7 +1329,7 @@ public void testRefreshMetric() throws IOException { } long refreshCount = shard.refreshStats().getTotal(); indexDoc(shard, "_doc", "test"); - try (Engine.GetResult ignored = shard.get(new Engine.Get(true, false, "test", "test", + try (Engine.GetResult ignored = shard.get(new Engine.Get(true, false, "_doc", "test", new Term(IdFieldMapper.NAME, Uid.encodeId("test"))))) { assertThat(shard.refreshStats().getTotal(), equalTo(refreshCount+1)); } @@ -2026,7 +2029,7 @@ public void testSearcherWrapperIsUsed() throws IOException { shard.refresh("test"); try (Engine.GetResult getResult = shard - .get(new Engine.Get(false, false, "test", "1", + .get(new Engine.Get(false, false, "_doc", "1", new Term(IdFieldMapper.NAME, Uid.encodeId("1"))))) { assertTrue(getResult.exists()); assertNotNull(getResult.searcher()); @@ -2068,7 +2071,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { assertEquals(search.totalHits.value, 1); } try (Engine.GetResult getResult = newShard - .get(new Engine.Get(false, false, "test", "1", + .get(new Engine.Get(false, false, "_doc", "1", new Term(IdFieldMapper.NAME, Uid.encodeId("1"))))) { assertTrue(getResult.exists()); assertNotNull(getResult.searcher()); // make sure get uses the wrapped reader @@ -3497,4 +3500,57 @@ public void testResetEngine() throws Exception { public Settings threadPoolSettings() { return Settings.builder().put(super.threadPoolSettings()).put("thread_pool.estimated_time_interval", "5ms").build(); } + + public void testDeleteOnWrongType() throws IOException { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetaData metaData = IndexMetaData.builder("index") + .putMapping("some_type", "{ \"properties\": {}}") + .settings(settings) + .build(); + IndexShard shard = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null); + recoverShardFromStore(shard); + Engine.IndexResult indexResult = indexDoc(shard, "some_type", "id", "{}"); + assertTrue(indexResult.isCreated()); + + DeleteResult deleteResult = shard.applyDeleteOperationOnPrimary(Versions.MATCH_ANY, "some_other_type", "id", VersionType.INTERNAL); + assertFalse(deleteResult.isFound()); + + deleteResult = shard.applyDeleteOperationOnPrimary(Versions.MATCH_ANY, "_doc", "id", VersionType.INTERNAL); + assertTrue(deleteResult.isFound()); + + closeShards(shard); + } + + public void testTypelessGet() throws IOException { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetaData metaData = IndexMetaData.builder("index") + .putMapping("some_type", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}") + .settings(settings) + .primaryTerm(0, 1).build(); + IndexShard shard = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null); + recoverShardFromStore(shard); + Engine.IndexResult indexResult = indexDoc(shard, "some_type", "0", "{\"foo\" : \"bar\"}"); + assertTrue(indexResult.isCreated()); + + org.elasticsearch.index.engine.Engine.GetResult getResult = shard.get( + new Engine.Get(true, true, "some_type", "0", new Term("_id", Uid.encodeId("0")))); + assertTrue(getResult.exists()); + getResult.close(); + + getResult = shard.get(new Engine.Get(true, true, "some_other_type", "0", new Term("_id", Uid.encodeId("0")))); + assertFalse(getResult.exists()); + getResult.close(); + + getResult = shard.get(new Engine.Get(true, true, "_doc", "0", new Term("_id", Uid.encodeId("0")))); + assertTrue(getResult.exists()); + getResult.close(); + + closeShards(shard); + } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index 04d15d39b58e9..7db904f89dfa8 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; @@ -77,4 +78,30 @@ public void testGetForUpdate() throws IOException { closeShards(primary); } + + public void testTypelessGetForUpdate() throws IOException { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetaData metaData = IndexMetaData.builder("index") + .putMapping("some_type", "{ \"properties\": { \"foo\": { \"type\": \"text\"}}}") + .settings(settings) + .primaryTerm(0, 1).build(); + IndexShard shard = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null); + recoverShardFromStore(shard); + Engine.IndexResult indexResult = indexDoc(shard, "some_type", "0", "{\"foo\" : \"bar\"}"); + assertTrue(indexResult.isCreated()); + + GetResult getResult = shard.getService().getForUpdate("some_type", "0", Versions.MATCH_ANY, VersionType.INTERNAL); + assertTrue(getResult.isExists()); + + getResult = shard.getService().getForUpdate("some_other_type", "0", Versions.MATCH_ANY, VersionType.INTERNAL); + assertFalse(getResult.isExists()); + + getResult = shard.getService().getForUpdate("_doc", "0", Versions.MATCH_ANY, VersionType.INTERNAL); + assertTrue(getResult.isExists()); + + closeShards(shard); + } }