Support for Elasticsearch v7 (#1216)

* Let elasticsearch track the actual number of hits. * Updated elasticsearch version on dependencies and containers. * Disbale dynamic field mapping in ES indices.
Materials-Consortia · Jun 9, 2022 · c57a0ac · c57a0ac
1 parent 1f02600
commit c57a0ac
Show file tree

Hide file tree

Showing 6 changed files with 161 additions and 163 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -162,12 +162,13 @@ jobs:
         ports:
         - 5432:5432
       elasticsearch:
-        image: elasticsearch:6.8.23
+        image: elasticsearch:7.17.1
         ports:
           - 9200:9200
           - 9300:9300
         env:
           discovery.type: single-node
+          xpack.security.enabled: 'false'
 
 
     steps:

diff --git a/INSTALL.md b/INSTALL.md
@@ -101,7 +101,7 @@ These commands should be run from a local optimade-python-tools directory.
 The following command starts a local Elasticsearch v6 instance, runs the test suite, then stops and deletes the containers (required as the tests insert some data):
 
 ```shell
-docker run -d --name elasticsearch_test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:6.8.23 \
+docker run -d --name elasticsearch_test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:7.17.1 \
 && sleep 10 \
 && OPTIMADE_DATABASE_BACKEND="elastic" py.test; \
 docker container stop elasticsearch_test; docker container rm elasticsearch_test

diff --git a/optimade/server/entry_collections/elastic_indexes.json b/optimade/server/entry_collections/elastic_indexes.json
@@ -1,158 +1,157 @@
 {
   "references": {
     "mappings": {
-      "doc": {
-        "properties": {
-          "address": {
-            "type": "keyword"
-          },
-          "annote": {
-            "type": "keyword"
-          },
-          "authors": {
-            "properties": {
-              "name": {
-                "type": "keyword"
-              },
-              "firstname": {
-                "type": "keyword"
-              },
-              "lastname": {
-                "type": "keyword"
-              }
+      "properties": {
+        "address": {
+          "type": "keyword"
+        },
+        "annote": {
+          "type": "keyword"
+        },
+        "authors": {
+          "properties": {
+            "name": {
+              "type": "keyword"
+            },
+            "firstname": {
+              "type": "keyword"
+            },
+            "lastname": {
+              "type": "keyword"
             }
-          },
-          "bib_type": {
-            "type": "keyword"
-          },
-          "booktitle": {
-            "type": "keyword"
-          },
-          "chapter": {
-            "type": "keyword"
-          },
-          "crossref": {
-            "type": "keyword"
-          },
-          "doi": {
-            "type": "keyword"
-          },
-          "edition": {
-            "type": "keyword"
-          },
-          "editors": {
-            "type": "keyword"
-          },
-          "howpublished": {
-            "type": "keyword"
-          },
-          "id": {
-            "type": "keyword"
-          },
-          "immutable_id": {
-            "type": "keyword"
-          },
-          "institution": {
-            "type": "keyword"
-          },
-          "journal": {
-            "type": "keyword"
-          },
-          "key": {
-            "type": "keyword"
-          },
-          "last_modified": {
-            "type": "date"
-          },
-          "links": {
-            "type": "keyword"
-          },
-          "month": {
-            "type": "keyword"
-          },
-          "note": {
-            "type": "keyword"
-          },
-          "number": {
-            "type": "keyword"
-          },
-          "organization": {
-            "type": "keyword"
-          },
-          "pages": {
-            "type": "keyword"
-          },
-          "publisher": {
-            "type": "keyword"
-          },
-          "relationships": {
-            "type": "keyword"
-          },
-          "school": {
-            "type": "keyword"
-          },
-          "series": {
-            "type": "keyword"
-          },
-          "title": {
-            "type": "keyword"
-          },
-          "type": {
-            "type": "keyword"
-          },
-          "url": {
-            "type": "keyword"
-          },
-          "volume": {
-            "type": "keyword"
-          },
-          "year": {
-            "type": "keyword"
           }
+        },
+        "bib_type": {
+          "type": "keyword"
+        },
+        "booktitle": {
+          "type": "keyword"
+        },
+        "chapter": {
+          "type": "keyword"
+        },
+        "crossref": {
+          "type": "keyword"
+        },
+        "doi": {
+          "type": "keyword"
+        },
+        "edition": {
+          "type": "keyword"
+        },
+        "editors": {
+          "type": "keyword"
+        },
+        "howpublished": {
+          "type": "keyword"
+        },
+        "id": {
+          "type": "keyword"
+        },
+        "immutable_id": {
+          "type": "keyword"
+        },
+        "institution": {
+          "type": "keyword"
+        },
+        "journal": {
+          "type": "keyword"
+        },
+        "key": {
+          "type": "keyword"
+        },
+        "last_modified": {
+          "type": "date"
+        },
+        "links": {
+          "type": "keyword"
+        },
+        "month": {
+          "type": "keyword"
+        },
+        "note": {
+          "type": "keyword"
+        },
+        "number": {
+          "type": "keyword"
+        },
+        "organization": {
+          "type": "keyword"
+        },
+        "pages": {
+          "type": "keyword"
+        },
+        "publisher": {
+          "type": "keyword"
+        },
+        "relationships": {
+          "type": "keyword"
+        },
+        "school": {
+          "type": "keyword"
+        },
+        "series": {
+          "type": "keyword"
+        },
+        "title": {
+          "type": "keyword"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "url": {
+          "type": "keyword"
+        },
+        "volume": {
+          "type": "keyword"
+        },
+        "year": {
+          "type": "keyword"
         }
       }
     }
   },
   "structures": {
     "mappings": {
-      "doc": {
-        "properties": {
-          "chemical_formula_anonymous": {
-            "type": "keyword"
-          },
-          "chemical_formula_descriptive": {
-            "type": "keyword"
-          },
-          "chemical_formula_hill": {
-            "type": "keyword"
-          },
-          "chemical_formula_reduced": {
-            "type": "keyword"
-          },
-          "dimension_types": {
-            "type": "integer"
-          },
-          "elements": {
-            "type": "keyword"
-          },
-          "elements_ratios": {
-            "type": "long"
-          },
-          "id": {
-            "type": "keyword"
-          },
-          "nelements": {
-            "type": "integer"
-          },
-          "nsites": {
-            "type": "integer"
-          },
-          "species_at_sites": {
-            "type": "keyword"
-          },
-          "structure_features": {
-            "type": "keyword"
-          }
+      "properties": {
+        "chemical_formula_anonymous": {
+          "type": "keyword"
+        },
+        "chemical_formula_descriptive": {
+          "type": "keyword"
+        },
+        "chemical_formula_hill": {
+          "type": "keyword"
+        },
+        "chemical_formula_reduced": {
+          "type": "keyword"
+        },
+        "dimension_types": {
+          "type": "integer"
+        },
+        "elements": {
+          "type": "keyword"
+        },
+        "elements_ratios": {
+          "type": "long"
+        },
+        "id": {
+          "type": "keyword"
+        },
+        "nelements": {
+          "type": "integer"
+        },
+        "nsites": {
+          "type": "integer"
+        },
+        "species_at_sites": {
+          "type": "keyword"
+        },
+        "structure_features": {
+          "type": "keyword"
+        },
+        "lattice_vectors": {
+          "type": "float"
         }
       }
     }

diff --git a/optimade/server/entry_collections/elasticsearch.py b/optimade/server/entry_collections/elasticsearch.py
@@ -66,11 +66,12 @@ def create_optimade_index(self) -> None:
             )
 
         properties = {}
-        for field in list(body["mappings"]["doc"]["properties"].keys()):
+        for field in list(body["mappings"]["properties"].keys()):
             properties[self.resource_mapper.get_backend_field(field)] = body[
                 "mappings"
-            ]["doc"]["properties"].pop(field)
-        body["mappings"]["doc"]["properties"] = properties
+            ]["properties"].pop(field)
+        properties["id"] = {"type": "keyword"}
+        body["mappings"]["properties"] = properties
         self.client.indices.create(index=self.name, body=body, ignore=400)
 
         LOGGER.debug(f"Created Elastic index for {self.name!r} with body {body}")
@@ -96,20 +97,16 @@ def create_elastic_index_from_mapper(
             The `body` parameter to pass to `client.indices.create(..., body=...)`.
 
         """
-        return {
-            "mappings": {
-                "doc": {
-                    "properties": {
-                        resource_mapper.get_optimade_field(field): {"type": "keyword"}
-                        for field in fields
-                    }
-                }
-            }
+        properties = {
+            resource_mapper.get_optimade_field(field): {"type": "keyword"}
+            for field in fields
         }
+        properties["id"] = {"type": "keyword"}
+        return {"mappings": {"properties": properties}}
 
     def __len__(self):
         """Returns the total number of entries in the collection."""
-        return Search(using=self.client, index=self.name).execute().hits.total
+        return Search(using=self.client, index=self.name).execute().hits.total.value
 
     def insert(self, data: List[EntryResource]) -> None:
         """Add the given entries to the underlying database.
@@ -142,7 +139,7 @@ def get_id(item):
                 {
                     "_index": self.name,
                     "_id": get_id(item),
-                    "_type": "doc",
+                    "_type": "_doc",
                     "_source": item,
                 }
                 for item in data
@@ -189,12 +186,13 @@ def _run_db_query(
         search = search.sort(*elastic_sort)
 
         search = search[page_offset : page_offset + limit]
+        search = search.extra(track_total_hits=True)
         response = search.execute()
 
         results = [hit.to_dict() for hit in response.hits]
 
         if not single_entry:
-            data_returned = response.hits.total
+            data_returned = response.hits.total.value
             more_data_available = page_offset + limit < data_returned
         else:
             # SingleEntryQueryParams, e.g., /structures/{entry_id}

diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-elasticsearch-dsl==6.4.0
+elasticsearch-dsl==7.4.0
 email_validator==1.2.1
 fastapi==0.78.0
 lark==0.12.0