Update Elastica for Elasticsearch 7 (#1563)

This PR is to update the compatibility for Elastic to Elasticsearch 7. The document followed to update Elastica is the one on [ES7 Breaking Changes](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html).
ruflin · Jun 4, 2019 · 6432042 · 6432042
1 parent 63553e4
commit 6432042
Show file tree

Hide file tree

Showing 41 changed files with 153 additions and 260 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,11 +6,17 @@ All notable changes to this project will be documented in this file based on the
 ### Backward Compatibility Breaks
 * \Elastica\AbstractUpdateAction::getOptions( $fields ) no longer supports the $underscore parameter, option names must match what elasticsearch expects.
 * Removed no longer supported \Elastica\Query\QueryString::setAutoGeneratePhraseQueries( $bool ) [#1622](https://github.com/ruflin/Elastica/pull/1622)
+* Replaced [params._agg](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#_replaced_literal_params__agg_literal_with_literal_state_literal_context_variable_in_scripted_metric_aggregations) with state context variable in scripted metric aggregations 
+* [Camel Case](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#_camel_case_and_underscore_parameters_deprecated_in_6_x_have_been_removed) and underscore parameters deprecated in 6.x have been removed
+* The parameter [fields](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#_the_parameter_literal_fields_literal_deprecated_in_6_x_has_been_removed_from_bulk_request) deprecated in 6.x has been removed from Bulk requestedit and Update request.
+* The [_parent](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-parent-field.html) field has been removed in favour of the join field.
+* hits.total is now an object in the search response [hits.total](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#_literal_hits_total_literal_is_now_an_object_in_the_search_response) 
 
 ### Bugfixes
 * Always set the Guzzle `base_uri` to support connecting to multiple ES hosts. [#1618](https://github.com/ruflin/Elastica/pull/1618)
 * Properly handle underscore prefixes in options and bulk request metadata ([cf upstream](https://github.com/elastic/elasticsearch/issues/26886). [#1621](https://github.com/ruflin/Elastica/pull/1621)
 * Preserve zeros while doing float serialization to JSON. [#1635](https://github.com/ruflin/Elastica/pull/1635)
+* Add ```settings``` level on json to create an Index in all tests (it worked till 6.x but it shouldn't work)
 
 ### Added
 
@@ -26,6 +32,7 @@ All notable changes to this project will be documented in this file based on the
 
 ### Added
 
+* The preferred type name is [_doc](https://www.elastic.co/guide/en/elasticsearch/reference/6.5/removal-of-types.html), so that index APIs have the same path as they will have in 7.0
 * Added `BucketSelector` aggregation [#1554](https://github.com/ruflin/Elastica/pull/1554)
 * Added `DerivativeAggregation` [#1553](https://github.com/ruflin/Elastica/pull/1553)
 * The preferred type name is [_doc](https://www.elastic.co/guide/en/elasticsearch/reference/6.5/removal-of-types.html), so that index APIs have the same path as they will have in 7.0
@@ -36,6 +43,9 @@ All notable changes to this project will be documented in this file based on the
 
 * Reduced memory footprint of response by not keeping the raw JSON data when JSON after JSON has been parsed. [#1588](https://github.com/ruflin/Elastica/pull/1588)
 
+### Deprecated
+* Index templates use index_patterns instead of [template](https://www.elastic.co/guide/en/elasticsearch/reference/6.5/breaking-changes-6.0.html#_index_templates_use_literal_index_patterns_literal_instead_of_literal_template_literal)
+
 ## [6.1.0](https://github.com/ruflin/Elastica/compare/6.0.2...6.1.0)
 
 ### Backward Compatibility Breaks

diff --git a/README.md b/README.md
@@ -31,7 +31,7 @@ This project tries to follow Elasticsearch in terms of [End of Life](https://www
 
 | Elastica                                                                                | ElasticSearch | elasticsearch-php | PHP      |
 | --------------------------------------------------------------------------------------- | ------------- | ----------------- | -------- |
-| [7.x](https://github.com/ruflin/Elastica/tree/master)                                   | 7.x           | ^6.0              | ^7.0     |
+| [7.x](https://github.com/ruflin/Elastica/tree/master)                                   | 7.x           | dev-master              | ^7.0     |
 | [6.x](https://github.com/ruflin/Elastica/tree/6.x)                                      | 6.x           | ^6.0              | ^7.0     |
 | [5.x](https://github.com/ruflin/Elastica/tree/5.x)                                      | 5.x           | ^5.0              | \>=5.6   |
 | [3.2.3](https://github.com/ruflin/Elastica/tree/3.2.3) (unmaintained)                   | 2.4.0         | no                | \>=5.4   |

diff --git a/composer.json b/composer.json
@@ -14,7 +14,7 @@
     "require": {
         "php": "^7.0",
         "psr/log": "~1.0",
-        "elasticsearch/elasticsearch": "^6.0"
+        "elasticsearch/elasticsearch": "dev-master"
     },
     "require-dev": {
         "aws/aws-sdk-php": "~3.0",

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,7 +10,7 @@ services:
     networks:
       - esnet
 
-  elasticsearch6_1:
+  elasticsearch7_1:
     build: ./env/elasticsearch/
     container_name: elasticsearch
     privileged: true
@@ -34,7 +34,7 @@ services:
     networks:
       esnet:
         ipv4_address: 10.10.10.15
-  elasticsearch6_2:
+  elasticsearch7_2:
     build: ./env/elasticsearch/
     privileged: true
     environment:

diff --git a/env/elasticsearch/Dockerfile b/env/elasticsearch/Dockerfile
@@ -1,5 +1,5 @@
 #https://www.docker.elastic.co/
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.6.1
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.1.0
 MAINTAINER Nicolas Ruflin <spam@ruflin.com>
 
 RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch ingest-attachment

diff --git a/env/elasticsearch/elasticsearch.yml b/env/elasticsearch/elasticsearch.yml
@@ -5,17 +5,15 @@ cluster.name: "docker-cluster-elastica"
 # Without this, travis builds will be failed with OutOfMemory error
 processors: 1
 
-#network.host: 0.0.0.0
-
-#script.allowed_types: inline, stored
-#script.allowed_contexts: search, update, aggs
-
-
 # minimum_master_nodes need to be explicitly set when bound on a public IP
 # set to 1 to allow single node clusters
 # Details: https://github.com/elastic/elasticsearch/pull/17288
-discovery.zen.minimum_master_nodes: 2
-discovery.zen.ping.unicast.hosts: ["elasticsearch"]
+discovery.seed_hosts:
+  - 10.10.10.15
+  - 10.10.10.16
+
+cluster.initial_master_nodes:
+  - 10.10.10.15
 
 # at the moment it's not possible to remove the x-pack plugin from the official ES docker image
 # https://github.com/elastic/elasticsearch-docker/issues/36

diff --git a/lib/Elastica/AbstractUpdateAction.php b/lib/Elastica/AbstractUpdateAction.php
@@ -165,38 +165,6 @@ public function hasVersionType()
         return $this->hasParam('version_type');
     }
 
-    /**
-     * Sets parent document id.
-     *
-     * @param string|int $parent Parent document id
-     *
-     * @return $this
-     *
-     * @see https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-parent-field.html
-     */
-    public function setParent($parent)
-    {
-        return $this->setParam('parent', $parent);
-    }
-
-    /**
-     * Returns the parent document id.
-     *
-     * @return string|int Parent document id
-     */
-    public function getParent()
-    {
-        return $this->getParam('parent');
-    }
-
-    /**
-     * @return bool
-     */
-    public function hasParent()
-    {
-        return $this->hasParam('parent');
-    }
-
     /**
      * Set operation type.
      *
@@ -246,15 +214,15 @@ public function setRouting($value)
      */
     public function getRouting()
     {
-        return $this->getParam('routing');
+        return $this->getParam('_routing');
     }
 
     /**
      * @return bool
      */
     public function hasRouting()
     {
-        return $this->hasParam('routing');
+        return $this->hasParam('_routing');
     }
 
     /**
@@ -318,7 +286,7 @@ public function getRetryOnConflict()
      */
     public function hasRetryOnConflict()
     {
-        return $this->hasParam('retry_on_conflict');
+        return $this->hasParam('_retry_on_conflict');
     }
 
     /**

diff --git a/lib/Elastica/Bulk/Action.php b/lib/Elastica/Bulk/Action.php
@@ -177,7 +177,7 @@ public function setId($id): self
      */
     public function setRouting($routing): self
     {
-        $this->_metadata['_routing'] = $routing;
+        $this->_metadata['routing'] = $routing;
 
         return $this;
     }

diff --git a/lib/Elastica/Client.php b/lib/Elastica/Client.php
@@ -398,7 +398,6 @@ public function updateDocument($id, $data, $index, $type, array $options = [])
                     'routing',
                     'percolate',
                     'parent',
-                    'fields',
                     'retry_on_conflict',
                     'consistency',
                     'replication',
@@ -407,13 +406,6 @@ public function updateDocument($id, $data, $index, $type, array $options = [])
                 ]
             );
             $options += $docOptions;
-            // set fields param to source only if options was not set before
-            if ($data instanceof Document && ($data->isAutoPopulate()
-                || $this->getConfigValue(['document', 'autoPopulate'], false))
-                && !isset($options['fields'])
-            ) {
-                $options['fields'] = '_source';
-            }
         } else {
             $requestData = $data;
         }
@@ -425,12 +417,6 @@ public function updateDocument($id, $data, $index, $type, array $options = [])
             }
         }
 
-        if (!isset($options['retry_on_conflict'])) {
-            if ($retryOnConflict = $this->getConfig('retryOnConflict')) {
-                $options['retry_on_conflict'] = $retryOnConflict;
-            }
-        }
-
         $endpoint->setBody($requestData);
         $endpoint->setParams($options);
 
@@ -444,40 +430,11 @@ public function updateDocument($id, $data, $index, $type, array $options = [])
             if (isset($responseData['_version'])) {
                 $data->setVersion($responseData['_version']);
             }
-            if (isset($options['fields'])) {
-                $this->_populateDocumentFieldsFromResponse($response, $data, $options['fields']);
-            }
         }
 
         return $response;
     }
 
-    /**
-     * @param \Elastica\Response $response
-     * @param \Elastica\Document $document
-     * @param string             $fields   Array of field names to be populated or '_source' if whole document data should be updated
-     */
-    protected function _populateDocumentFieldsFromResponse(Response $response, Document $document, $fields)
-    {
-        $responseData = $response->getData();
-        if ('_source' == $fields) {
-            if (isset($responseData['get']['_source']) && \is_array($responseData['get']['_source'])) {
-                $document->setData($responseData['get']['_source']);
-            }
-        } else {
-            $keys = \explode(',', $fields);
-            $data = $document->getData();
-            foreach ($keys as $key) {
-                if (isset($responseData['get']['fields'][$key])) {
-                    $data[$key] = $responseData['get']['fields'][$key];
-                } elseif (isset($data[$key])) {
-                    unset($data[$key]);
-                }
-            }
-            $document->setData($data);
-        }
-    }
-
     /**
      * Bulk deletes documents.
      *
@@ -689,7 +646,6 @@ public function request($path, $method = Request::GET, $data = [], array $query
             $response = $this->_lastResponse = $request->send();
         } catch (ConnectionException $e) {
             $this->_connectionPool->onFail($connection, $e, $this);
-
             $this->_log($e);
 
             // In case there is no valid connection left, throw exception which caused the disabling of the connection.
@@ -756,23 +712,6 @@ protected function _log($context)
         ]);
     }
 
-    /**
-     * Optimizes all search indices.
-     *
-     * @param array $args OPTIONAL Optional arguments
-     *
-     * @return \Elastica\Response Response object
-     *
-     * @deprecated Replaced by forcemergeAll
-     * @see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-optimize.html
-     */
-    public function optimizeAll($args = [])
-    {
-        \trigger_error('Deprecated: Elastica\Client::optimizeAll() is deprecated and will be removed in further Elastica releases. Use Elastica\Client::forcemergeAll() instead.', E_USER_DEPRECATED);
-
-        return $this->forcemergeAll($args);
-    }
-
     /**
      * Force merges all search indices.
      *

diff --git a/lib/Elastica/Index.php b/lib/Elastica/Index.php
@@ -581,7 +581,11 @@ public function request($path, $method, $data = [], array $query = [])
     public function requestEndpoint(AbstractEndpoint $endpoint)
     {
         $cloned = clone $endpoint;
+
         $cloned->setIndex($this->getName());
+        if ($endpoint instanceof Create || $endpoint instanceof \Elasticsearch\Endpoints\Indices\Mapping\Put) {
+            $cloned->setParams(['include_type_name' => true]);
+        }
 
         return $this->getClient()->requestEndpoint($cloned);
     }

diff --git a/lib/Elastica/Index/Settings.php b/lib/Elastica/Index/Settings.php
@@ -26,7 +26,7 @@ class Settings
 
     const DEFAULT_NUMBER_OF_REPLICAS = 1;
 
-    const DEFAULT_NUMBER_OF_SHARDS = 5;
+    const DEFAULT_NUMBER_OF_SHARDS = 1;
 
     /**
      * Response.

diff --git a/lib/Elastica/ResultSet.php b/lib/Elastica/ResultSet.php
@@ -157,7 +157,7 @@ public function getTotalHits()
     {
         $data = $this->_response->getData();
 
-        return (int) ($data['hits']['total'] ?? 0);
+        return (int) ($data['hits']['total']['value'] ?? 0);
     }
 
     /**

diff --git a/lib/Elastica/Type.php b/lib/Elastica/Type.php
@@ -314,9 +314,9 @@ public function setMapping($mapping, array $query = [])
      *
      * @return array Current mapping
      */
-    public function getMapping()
+    public function getMapping($includeTypeName = true)
     {
-        $response = $this->requestEndpoint(new Get());
+        $response = $this->requestEndpoint(new Get(), $includeTypeName);
         $data = $response->getData();
 
         $mapping = \array_shift($data);
@@ -396,7 +396,7 @@ public function deleteDocument(Document $document)
     {
         $options = $document->getOptions(
             [
-                'version',
+                '_version',
                 'version_type',
                 'routing',
                 'parent',
@@ -520,10 +520,13 @@ public function request($path, $method, $data = [], array $query = [])
      *
      * @return Response
      */
-    public function requestEndpoint(AbstractEndpoint $endpoint)
+    public function requestEndpoint(AbstractEndpoint $endpoint, $includeTypeName = false)
     {
         $cloned = clone $endpoint;
         $cloned->setType($this->getName());
+        if ($includeTypeName) {
+            $cloned->setParams(['include_type_name' => true]);
+        }
 
         return $this->getIndex()->requestEndpoint($cloned);
     }

diff --git a/test/Elastica/Aggregation/ChildrenTest.php b/test/Elastica/Aggregation/ChildrenTest.php
@@ -15,7 +15,7 @@ protected function _getIndexForTest(): Index
     {
         $client = $this->_getClient();
         $index = $client->getIndex('testaggregationchildren');
-        $index->create(['index' => ['number_of_shards' => 2, 'number_of_replicas' => 1]], true);
+        $index->create(['settings' => ['index' => ['number_of_shards' => 2, 'number_of_replicas' => 1]]], true);
 
         $type = $index->getType(\strtolower(
             'typechildren'.\uniqid()

diff --git a/test/Elastica/Aggregation/ParentAggregationTest.php b/test/Elastica/Aggregation/ParentAggregationTest.php
@@ -14,7 +14,7 @@ protected function _getIndexForTest()
     {
         $client = $this->_getClient();
         $index = $client->getIndex('testaggregationparent');
-        $index->create(['index' => ['number_of_shards' => 2, 'number_of_replicas' => 1]], true);
+        $index->create(['settings' => ['index' => ['number_of_shards' => 2, 'number_of_replicas' => 1]]], true);
 
         $type = $index->getType(\strtolower(
             'typeparent'.\uniqid()

diff --git a/test/Elastica/Aggregation/ScriptedMetricTest.php b/test/Elastica/Aggregation/ScriptedMetricTest.php
@@ -38,9 +38,10 @@ public function testScriptedMetricAggregation()
     {
         $agg = new ScriptedMetric(
             'scripted',
-            'params._agg.durations = []',
-            'params._agg.durations.add(doc.end.value - doc.start.value)',
-            'return params._agg.durations'
+            'state.durations = []',
+            'state.durations.add(doc.end.value - doc.start.value)',
+            'return state.durations',
+            'return states'
         );
 
         $query = new Query();

diff --git a/test/Elastica/Base.php b/test/Elastica/Base.php
@@ -129,7 +129,7 @@ protected function _createIndex($name = null, $delete = true, $shards = 1)
         $index = $client->getIndex('elastica_'.$name);
 
         if ('elasticsearch' === \getenv('ES_HOST')) {
-            $index->create(['index' => ['number_of_shards' => $shards, 'number_of_replicas' => 1]], $delete);
+            $index->create(['settings' => ['index' => ['number_of_shards' => $shards, 'number_of_replicas' => 1]]], $delete);
         }
 
         return $index;