Skip to content

Commit

Permalink
Disbale dynamic field mapping in ES indices.
Browse files Browse the repository at this point in the history
  • Loading branch information
@markus1978 @ml-evs
markus1978 authored and ml-evs committed Jun 9, 2022
1 parent d4281dd commit 3ae2850
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 157 deletions.
287 changes: 143 additions & 144 deletions optimade/server/entry_collections/elastic_indexes.json
View file
Original file line number Diff line number Diff line change
@@ -1,158 +1,157 @@
{
"references": {
"mappings": {
"doc": {
"properties": {
"address": {
"type": "keyword"
},
"annote": {
"type": "keyword"
},
"authors": {
"properties": {
"name": {
"type": "keyword"
},
"firstname": {
"type": "keyword"
},
"lastname": {
"type": "keyword"
}
"properties": {
"address": {
"type": "keyword"
},
"annote": {
"type": "keyword"
},
"authors": {
"properties": {
"name": {
"type": "keyword"
},
"firstname": {
"type": "keyword"
},
"lastname": {
"type": "keyword"
}
},
"bib_type": {
"type": "keyword"
},
"booktitle": {
"type": "keyword"
},
"chapter": {
"type": "keyword"
},
"crossref": {
"type": "keyword"
},
"doi": {
"type": "keyword"
},
"edition": {
"type": "keyword"
},
"editors": {
"type": "keyword"
},
"howpublished": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"immutable_id": {
"type": "keyword"
},
"institution": {
"type": "keyword"
},
"journal": {
"type": "keyword"
},
"key": {
"type": "keyword"
},
"last_modified": {
"type": "date"
},
"links": {
"type": "keyword"
},
"month": {
"type": "keyword"
},
"note": {
"type": "keyword"
},
"number": {
"type": "keyword"
},
"organization": {
"type": "keyword"
},
"pages": {
"type": "keyword"
},
"publisher": {
"type": "keyword"
},
"relationships": {
"type": "keyword"
},
"school": {
"type": "keyword"
},
"series": {
"type": "keyword"
},
"title": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"url": {
"type": "keyword"
},
"volume": {
"type": "keyword"
},
"year": {
"type": "keyword"
}
},
"bib_type": {
"type": "keyword"
},
"booktitle": {
"type": "keyword"
},
"chapter": {
"type": "keyword"
},
"crossref": {
"type": "keyword"
},
"doi": {
"type": "keyword"
},
"edition": {
"type": "keyword"
},
"editors": {
"type": "keyword"
},
"howpublished": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"immutable_id": {
"type": "keyword"
},
"institution": {
"type": "keyword"
},
"journal": {
"type": "keyword"
},
"key": {
"type": "keyword"
},
"last_modified": {
"type": "date"
},
"links": {
"type": "keyword"
},
"month": {
"type": "keyword"
},
"note": {
"type": "keyword"
},
"number": {
"type": "keyword"
},
"organization": {
"type": "keyword"
},
"pages": {
"type": "keyword"
},
"publisher": {
"type": "keyword"
},
"relationships": {
"type": "keyword"
},
"school": {
"type": "keyword"
},
"series": {
"type": "keyword"
},
"title": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"url": {
"type": "keyword"
},
"volume": {
"type": "keyword"
},
"year": {
"type": "keyword"
}
}
}
},
"structures": {
"mappings": {
"doc": {
"properties": {
"chemical_formula_anonymous": {
"type": "keyword"
},
"chemical_formula_descriptive": {
"type": "keyword"
},
"chemical_formula_hill": {
"type": "keyword"
},
"chemical_formula_reduced": {
"type": "keyword"
},
"dimension_types": {
"type": "integer"
},
"elements": {
"type": "keyword"
},
"elements_ratios": {
"type": "long"
},
"id": {
"type": "keyword"
},
"nelements": {
"type": "integer"
},
"nsites": {
"type": "integer"
},
"species_at_sites": {
"type": "keyword"
},
"structure_features": {
"type": "keyword"
}
"properties": {
"chemical_formula_anonymous": {
"type": "keyword"
},
"chemical_formula_descriptive": {
"type": "keyword"
},
"chemical_formula_hill": {
"type": "keyword"
},
"chemical_formula_reduced": {
"type": "keyword"
},
"dimension_types": {
"type": "integer"
},
"elements": {
"type": "keyword"
},
"elements_ratios": {
"type": "long"
},
"id": {
"type": "keyword"
},
"nelements": {
"type": "integer"
},
"nsites": {
"type": "integer"
},
"species_at_sites": {
"type": "keyword"
},
"structure_features": {
"type": "keyword"
},
"lattice_vectors": {
"type": "float"
}
}
}
Expand Down
23 changes: 10 additions & 13 deletions optimade/server/entry_collections/elasticsearch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,12 @@ def create_optimade_index(self) -> None:
)

properties = {}
for field in list(body["mappings"]["doc"]["properties"].keys()):
for field in list(body["mappings"]["properties"].keys()):
properties[self.resource_mapper.get_backend_field(field)] = body[
"mappings"
]["doc"]["properties"].pop(field)
body["mappings"]["doc"]["properties"] = properties
]["properties"].pop(field)
properties["id"] = {"type": "keyword"}
body["mappings"]["properties"] = properties
self.client.indices.create(index=self.name, body=body, ignore=400)

LOGGER.debug(f"Created Elastic index for {self.name!r} with body {body}")
Expand All @@ -96,16 +97,12 @@ def create_elastic_index_from_mapper(
The `body` parameter to pass to `client.indices.create(..., body=...)`.
"""
return {
"mappings": {
"doc": {
"properties": {
resource_mapper.get_optimade_field(field): {"type": "keyword"}
for field in fields
}
}
}
properties = {
resource_mapper.get_optimade_field(field): {"type": "keyword"}
for field in fields
}
properties["id"] = {"type": "keyword"}
return {"mappings": {"properties": properties}}

def __len__(self):
"""Returns the total number of entries in the collection."""
Expand Down Expand Up @@ -142,7 +139,7 @@ def get_id(item):
{
"_index": self.name,
"_id": get_id(item),
"_type": "doc",
"_type": "_doc",
"_source": item,
}
for item in data
Expand Down

0 comments on commit 3ae2850

Please sign in to comment.