From 9e4c710209de16bd10a511559a586bfd2c7735bb Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Thu, 28 Jan 2016 21:43:19 +0100 Subject: [PATCH 01/41] TASK: Decouple the query from the request builder This change replace the request array by an OO implementation. This can be use to override the default query for doing function scoring or more complexe compound query. This change also introduce a FunctionScore query implementation. You can override the default Query implementation in your ```Objects.yaml```. Your own implementation must implement the ```QueryInterface```. You can use ```AbstractQuery``` to use some helpers methods. --- .../Domain/Model/AbstractQuery.php | 254 ++++++++++++++++++ .../Domain/Model/FilteredQuery.php | 121 +++++++++ .../Domain/Model/FunctionScoreQuery.php | 121 +++++++++ .../Domain/Model/QueryInterface.php | 144 ++++++++++ .../Eel/ElasticSearchQueryBuilder.php | 201 +++----------- .../Eel/ElasticSearchQueryBuilderTest.php | 20 +- 6 files changed, 692 insertions(+), 169 deletions(-) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php new file mode 100644 index 00000000..c4515463 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php @@ -0,0 +1,254 @@ +queryBuilder = $queryBuilder; + if ($request !== null) { + $this->request = $request; + } + } + + /** + * {@inheritdoc} + */ + public function toArray() + { + return $this->prepareRequest(); + } + + /** + * {@inheritdoc} + */ + public function getRequestAsJSON() + { + return json_encode($this); + } + + /** + * {@inheritdoc} + */ + public function addSortFilter($configuration) + { + if (!isset($this->request['sort'])) { + $this->request['sort'] = []; + } + $this->request['sort'][] = $configuration; + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function aggregation($name, array $aggregationDefinition, $parentPath = null) + { + if (!array_key_exists('aggregations', $this->request)) { + $this->request['aggregations'] = []; + } + + if ($parentPath !== null) { + $this->addSubAggregation($parentPath, $name, $aggregationDefinition); + } else { + $this->request['aggregations'][$name] = $aggregationDefinition; + } + + return $this->queryBuilder; + } + + /** + * This is an low level method for internal usage. + * + * You can add a custom $aggregationConfiguration under a given $parentPath. The $parentPath foo.bar would + * insert your $aggregationConfiguration under + * $this->request['aggregations']['foo']['aggregations']['bar']['aggregations'][$name] + * + * @param $parentPath + * @param $name + * @param array $aggregationConfiguration + * @return QueryInterface + * @throws Exception\QueryBuildingException + */ + protected function addSubAggregation($parentPath, $name, $aggregationConfiguration) + { + // Find the parentPath + $path =& $this->request['aggregations']; + + foreach (explode(".", $parentPath) as $subPart) { + if ($path == null || !array_key_exists($subPart, $path)) { + throw new Exception\QueryBuildingException("The parent path " . $subPart . " could not be found when adding a sub aggregation"); + } + $path =& $path[$subPart]['aggregations']; + } + + $path[$name] = $aggregationConfiguration; + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function suggestions($name, array $suggestionDefinition) + { + if (!array_key_exists('suggest', $this->request)) { + $this->request['suggest'] = []; + } + + $this->request['suggest'][$name] = $suggestionDefinition; + + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function highlight($fragmentSize, $fragmentCount = null) + { + if ($fragmentSize === false) { + // Highlighting is disabled. + unset($this->request['highlight']); + } else { + $this->request['highlight'] = [ + 'fields' => [ + '__fulltext*' => [ + 'fragment_size' => $fragmentSize, + 'no_match_size' => $fragmentSize, + 'number_of_fragments' => $fragmentCount + ] + ] + ]; + } + + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function setValueByPath($path, $value) + { + $this->request = Arrays::setValueByPath($this->request, $path, $value); + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function appendAtPath($path, array $data) + { + $currentElement =& $this->request; + foreach (explode('.', $path) as $pathPart) { + if (!isset($currentElement[$pathPart])) { + throw new Exception\QueryBuildingException('The element at path "' . $path . '" was not an array (failed at "' . $pathPart . '").', 1383716367); + } + $currentElement =& $currentElement[$pathPart]; + } + $currentElement[] = $data; + + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function jsonSerialize() { + return $this->prepareRequest(); + } + + /** + * {@inheritdoc} + */ + public function offsetSet($offset, $value) { + if (is_null($offset)) { + $this->request[] = $value; + } else { + $this->request[$offset] = $value; + } + } + + /** + * {@inheritdoc} + */ + public function offsetExists($offset) { + return isset($this->request[$offset]); + } + + /** + * {@inheritdoc} + */ + public function offsetUnset($offset) { + unset($this->request[$offset]); + } + + /** + * {@inheritdoc} + */ + public function offsetGet($offset) { + return isset($this->request[$offset]) ? $this->request[$offset] : null; + } + + /** + * Prepare the final request array + * + * This method is useful if you extend the current query implementation. + * + * @return array + */ + protected function prepareRequest() + { + return $this->request; + } + + /** + * All methods are considered safe + * + * @param string $methodName + * @return boolean + */ + public function allowsCallOfMethod($methodName) + { + return true; + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php new file mode 100644 index 00000000..7811ea61 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php @@ -0,0 +1,121 @@ + [ + 'filtered' => [ + 'query' => [ + 'bool' => [ + 'must' => [ + [ + 'match_all' => [] + ] + ] + ] + + ], + 'filter' => [ + 'bool' => [ + 'must' => [], + 'should' => [], + 'must_not' => [ + [ + 'term' => ['_hidden' => true] + ], + [ + 'range' => [ + '_hiddenBeforeDateTime' => [ + 'gt' => 'now' + ] + ] + ], + [ + 'range' => [ + '_hiddenAfterDateTime' => [ + 'lt' => 'now' + ] + ] + ], + ], + ] + ] + ] + ], + 'fields' => ['__path'] + ]; + + /** + * {@inheritdoc} + */ + public function getCountRequestAsJSON() + { + $request = $this->request; + foreach ($this->unsupportedFieldsInCountRequest as $field) { + if (isset($request[$field])) { + unset($request[$field]); + } + } + return json_encode($request); + } + + /** + * {@inheritdoc} + */ + public function size($size) + { + $this->request['size'] = (integer)$size; + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function from($size) + { + $this->request['from'] = (integer)$size; + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function fulltext($searchWord) + { + $this->appendAtPath('query.filtered.query.bool.must', [ + 'query_string' => [ + 'query' => $searchWord + ] + ]); + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + public function queryFilter($filterType, $filterOptions, $clauseType = 'must') + { + if (!in_array($clauseType, ['must', 'should', 'must_not'])) { + throw new Exception\QueryBuildingException('The given clause type "' . $clauseType . '" is not supported. Must be one of "must", "should", "must_not".', 1383716082); + } + return $this->appendAtPath('query.filtered.filter.bool.' . $clauseType, [$filterType => $filterOptions]); + } + +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php new file mode 100644 index 00000000..637e608c --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php @@ -0,0 +1,121 @@ + [] + ]; + + /** + * @param array $functions + * @return QueryBuilderInterface + */ + public function functions(array $functions) + { + if (isset($functions['functions'])) { + $this->functionScoreRequest = $functions; + } else { + $this->functionScoreRequest['functions'] = $functions; + } + return $this->queryBuilder; + } + + /** + * @param string $scoreMode + * @return QueryBuilderInterface + * @throws Exception\QueryBuildingException + */ + public function scoreMode($scoreMode) + { + if (!in_array($scoreMode, ['multiply', 'first', 'sum', 'avg', 'max', 'min'])) { + throw new Exception\QueryBuildingException('Invalid score mode', 1454016230); + } + $this->functionScoreRequest['score_mode'] = $scoreMode; + return $this->queryBuilder; + } + + /** + * @param string $boostMode + * @return QueryBuilderInterface + * @throws Exception\QueryBuildingException + */ + public function boostMode($boostMode) + { + if (!in_array($boostMode, ['multiply', 'replace', 'sum', 'avg', 'max', 'min'])) { + throw new Exception\QueryBuildingException('Invalid boost mode', 1454016229); + } + $this->functionScoreRequest['boost_mode'] = $boostMode; + return $this->queryBuilder; + } + + /** + * @param integer|float $boost + * @return QueryBuilderInterface + * @throws Exception\QueryBuildingException + */ + public function maxBoost($boost) + { + if (!is_numeric($boost)) { + throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); + } + $this->functionScoreRequest['max_boost'] = $boost; + return $this->queryBuilder; + } + + /** + * @param integer|float $score + * @return QueryBuilderInterface + * @throws Exception\QueryBuildingException + */ + public function minScore($score) + { + if (!is_numeric($score)) { + throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); + } + $this->functionScoreRequest['min_score'] = $score; + return $this->queryBuilder; + } + + /** + * {@inheritdoc} + */ + protected function prepareRequest() + { + if ($this->functionScoreRequest['functions'] === []) { + return parent::prepareRequest(); + } + $currentQuery = $this->request['query']; + + $baseQuery = $this->request; + unset($baseQuery['query']); + + $functionScore = $this->functionScoreRequest; + $functionScore['query'] = $currentQuery; + $query = Arrays::arrayMergeRecursiveOverrule($baseQuery, [ + 'query' => [ + 'function_score' => $functionScore + ] + ]); + return $query; + } + + +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php new file mode 100644 index 00000000..8d03cf0d --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php @@ -0,0 +1,144 @@ +request. + * + * Low-level method to manipulate the ElasticSearch Query + * + * @param string $path + * @param array $data + * @throws Exception\QueryBuildingException + * @return QueryBuilderInterface + */ + public function appendAtPath($path, array $data); +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php index 1c4009f8..540fd416 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php @@ -11,9 +11,13 @@ * The TYPO3 project - inspiring people to share! * * */ +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\QueryInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\QueryBuildingException; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; use TYPO3\Eel\ProtectedContextAwareInterface; use TYPO3\Flow\Annotations as Flow; +use TYPO3\Flow\Object\ObjectManagerInterface; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\TYPO3CR\Search\Search\QueryBuilderInterface; @@ -24,10 +28,16 @@ class ElasticSearchQueryBuilder implements QueryBuilderInterface, ProtectedConte { /** * @Flow\Inject - * @var \Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient + * @var ElasticSearchClient */ protected $elasticSearchClient; + /** + * @Flow\Inject + * @var ObjectManagerInterface + */ + protected $objectManager; + /** * The node inside which searching should happen * @@ -37,7 +47,7 @@ class ElasticSearchQueryBuilder implements QueryBuilderInterface, ProtectedConte /** * @Flow\Inject - * @var \Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface + * @var LoggerInterface */ protected $logger; @@ -61,13 +71,6 @@ class ElasticSearchQueryBuilder implements QueryBuilderInterface, ProtectedConte */ protected $from; - /** - * These fields are not accepted in a count request and must therefore be removed before doing so - * - * @var array - */ - protected $unsupportedFieldsInCountRequest = array('fields', 'sort', 'from', 'size', 'highlight', 'aggs', 'aggregations'); - /** * This (internal) array stores, for the last search request, a mapping from Node Identifiers * to the full ElasticSearch Hit which was returned. @@ -78,69 +81,24 @@ class ElasticSearchQueryBuilder implements QueryBuilderInterface, ProtectedConte */ protected $elasticSearchHitsIndexedByNodeFromLastRequest; - /** * The ElasticSearch request, as it is being built up. - * @var array + * @var QueryInterface */ - protected $request = array( - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-query.html - 'query' => array( - // The top-level query we're working on is a *filtered* query, as this allows us to efficiently - // apply *global constraints* in the form of *filters* which apply on the whole query. - // - // NOTE: we do NOT add a search request FILTER to the query currently, because that would mean - // that the filters ONLY apply for query results, but NOT for facet calculation (as explained on - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-filter.html) - // - // Reference: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-filtered-query.html - 'filtered' => array( - 'query' => array( - 'bool' => array( - 'must' => array( - array( - 'match_all' => array() - ) - ) - ) - - ), - 'filter' => array( - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-bool-filter.html - 'bool' => array( - 'must' => array(), - 'should' => array(), - 'must_not' => array( - // Filter out all hidden elements - array( - 'term' => array('_hidden' => true) - ), - // if now < hiddenBeforeDateTime: HIDE - // -> hiddenBeforeDateTime > now - array( - 'range' => array('_hiddenBeforeDateTime' => array( - 'gt' => 'now' - )) - ), - array( - 'range' => array('_hiddenAfterDateTime' => array( - 'lt' => 'now' - )) - ), - ), - ) - ) - ) - ), - 'fields' => array('__path') - ); - + protected $request; /** * @var array */ protected $result = array(); + /** + * Initialize Object + */ + public function initializeObject() + { + $this->request = $this->objectManager->get('Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\QueryInterface', $this); + } /** * HIGH-LEVEL API @@ -207,11 +165,7 @@ public function sortAsc($propertyName) */ public function sort($configuration) { - if (!isset($this->request['sort'])) { - $this->request['sort'] = array(); - } - - $this->request['sort'][] = $configuration; + $this->request->addSortFilter($configuration); return $this; } @@ -245,7 +199,7 @@ public function limit($limit) $this->limit = $limit; // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-from-size.html - $this->request['size'] = $limit * $currentWorkspaceNestingLevel; + $this->request->size($limit * $currentWorkspaceNestingLevel); return $this; } @@ -265,7 +219,7 @@ public function from($from) } $this->from = $from; - $this->request['from'] = $from; + $this->request->from($from); return $this; } @@ -356,10 +310,8 @@ public function lessThanOrEqual($propertyName, $value) */ public function queryFilter($filterType, $filterOptions, $clauseType = 'must') { - if (!in_array($clauseType, array('must', 'should', 'must_not'))) { - throw new QueryBuildingException('The given clause type "' . $clauseType . '" is not supported. Must be one of "must", "should", "must_not".', 1383716082); - } - return $this->appendAtPath('query.filtered.filter.bool.' . $clauseType, array($filterType => $filterOptions)); + $this->request->queryFilter($filterType, $filterOptions, $clauseType); + return $this; } /** @@ -374,15 +326,7 @@ public function queryFilter($filterType, $filterOptions, $clauseType = 'must') */ public function appendAtPath($path, array $data) { - $currentElement =& $this->request; - foreach (explode('.', $path) as $pathPart) { - if (!isset($currentElement[$pathPart])) { - throw new QueryBuildingException('The element at path "' . $path . '" was not an array (failed at "' . $pathPart . '").', 1383716367); - } - $currentElement =& $currentElement[$pathPart]; - } - $currentElement[] = $data; - + $this->request->appendAtPath($path, $data); return $this; } @@ -470,44 +414,7 @@ public function fieldBasedAggregation($name, $field, $type = "terms", $parentPat */ public function aggregation($name, array $aggregationDefinition, $parentPath = null) { - if (!array_key_exists("aggregations", $this->request)) { - $this->request['aggregations'] = array(); - } - - if ($parentPath !== null) { - $this->addSubAggregation($parentPath, $name, $aggregationDefinition); - } else { - $this->request['aggregations'][$name] = $aggregationDefinition; - } - - return $this; - } - - /** - * This is an low level method for internal usage. - * You can add a custom $aggregationConfiguration under a given $parentPath. The $parentPath foo.bar would - * insert your $aggregationConfiguration under - * $this->request['aggregations']['foo']['aggregations']['bar']['aggregations'][$name] - * - * @param $parentPath - * @param $name - * @param array $aggregationConfiguration - * @return $this - * @throws QueryBuildingException - */ - protected function addSubAggregation($parentPath, $name, $aggregationConfiguration) - { - // Find the parentPath - $path =& $this->request['aggregations']; - - foreach (explode(".", $parentPath) as $subPart) { - if ($path == null || !array_key_exists($subPart, $path)) { - throw new QueryBuildingException("The parent path ".$subPart." could not be found when adding a sub aggregation"); - } - $path =& $path[$subPart]['aggregations']; - } - - $path[$name] = $aggregationConfiguration; + $this->request->aggregation($name, $aggregationDefinition, $parentPath); return $this; } @@ -560,19 +467,14 @@ public function termSuggestions($text, $field = '_all', $name = 'suggestions') */ public function suggestions($name, array $suggestionDefinition) { - if (!array_key_exists('suggest', $this->request)) { - $this->request['suggest'] = []; - } - - $this->request['suggest'][$name] = $suggestionDefinition; - + $this->request->suggestions($name, $suggestionDefinition); return $this; } /** * Get the ElasticSearch request as we need it * - * @return array + * @return QueryInterface */ public function getRequest() { @@ -644,7 +546,8 @@ public function getFullElasticSearchHitForNode(NodeInterface $node) public function fetch() { $timeBefore = microtime(true); - $response = $this->elasticSearchClient->getIndex()->request('GET', '/_search', array(), json_encode($this->request)); + $request = $this->request->getRequestAsJSON(); + $response = $this->elasticSearchClient->getIndex()->request('GET', '/_search', array(), $request); $timeAfterwards = microtime(true); $this->result = $response->getTreatedContent(); @@ -652,7 +555,7 @@ public function fetch() $this->result['nodes'] = array(); if ($this->logThisQuery === true) { $this->logger->log(sprintf('Query Log (%s): %s -- execution time: %s ms -- Limit: %s -- Number of results returned: %s -- Total Results: %s', - $this->logMessage, json_encode($this->request), (($timeAfterwards - $timeBefore) * 1000), $this->limit, count($this->result['hits']['hits']), $this->result['hits']['total']), LOG_DEBUG); + $this->logMessage, $request, (($timeAfterwards - $timeBefore) * 1000), $this->limit, count($this->result['hits']['hits']), $this->result['hits']['total']), LOG_DEBUG); } if (array_key_exists('hits', $this->result) && is_array($this->result['hits']) && count($this->result['hits']) > 0) { $this->result['nodes'] = $this->convertHitsToNodes($this->result['hits']); @@ -684,21 +587,16 @@ public function execute() public function count() { $timeBefore = microtime(true); - $request = $this->request; - foreach ($this->unsupportedFieldsInCountRequest as $field) { - if (isset($request[$field])) { - unset($request[$field]); - } - } + $request = $this->getRequest()->getCountRequestAsJSON(); - $response = $this->elasticSearchClient->getIndex()->request('GET', '/_count', array(), json_encode($request)); + $response = $this->elasticSearchClient->getIndex()->request('GET', '/_count', array(), $request); $timeAfterwards = microtime(true); $treatedContent = $response->getTreatedContent(); $count = $treatedContent['count']; if ($this->logThisQuery === true) { - $this->logger->log('Count Query Log (' . $this->logMessage . '): ' . json_encode($this->request) . ' -- execution time: ' . (($timeAfterwards - $timeBefore) * 1000) . ' ms -- Total Results: ' . $count, LOG_DEBUG); + $this->logger->log('Count Query Log (' . $this->logMessage . '): ' . $request . ' -- execution time: ' . (($timeAfterwards - $timeBefore) * 1000) . ' ms -- Total Results: ' . $count, LOG_DEBUG); } return $count; @@ -713,14 +611,12 @@ public function count() */ public function fulltext($searchWord) { - $this->appendAtPath('query.filtered.query.bool.must', array( - 'query_string' => array( - 'query' => $searchWord - ) - )); - // We automatically enable result highlighting when doing fulltext searches. It is up to the user to use this information or not use it. - return $this->highlight(150, 2); + $this->request + ->fulltext($searchWord) + ->highlight(150, 2); + + return $this; } /** @@ -734,20 +630,7 @@ public function fulltext($searchWord) */ public function highlight($fragmentSize, $fragmentCount = null) { - if ($fragmentSize === false) { - // Highlighting is disabled. - unset($this->request['highlight']); - } else { - $this->request['highlight'] = array( - 'fields' => array( - '__fulltext*' => array( - 'fragment_size' => $fragmentSize, - 'no_match_size' => $fragmentSize, - 'number_of_fragments' => $fragmentCount - ) - ) - ); - } + $this->request->highlight($fragmentSize, $fragmentCount); return $this; } diff --git a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php index 2555f4e2..3980d9ad 100644 --- a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php +++ b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php @@ -99,7 +99,7 @@ public function basicRequestStructureTakesContextNodeIntoAccount() ), 'fields' => array('__path') ); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertSame($expected, $actual); } @@ -123,7 +123,7 @@ public function nodeTypeFilterWorks() '__typeAndSupertypes' => 'Foo.Bar:Baz' ) ); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertInArray($expected, $actual['query']['filtered']['filter']['bool']['must']); } @@ -138,7 +138,7 @@ public function sortAscWorks() 'fieldName' => array('order' => 'asc') ) ); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertSame($expected, $actual['sort']); } @@ -159,7 +159,7 @@ public function sortingIsAdditive() 'field3' => array('order' => 'asc') ) ); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertSame($expected, $actual['sort']); } @@ -169,7 +169,7 @@ public function sortingIsAdditive() public function limitWorks() { $this->queryBuilder->limit(2); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertSame(2, $actual['size']); } @@ -184,7 +184,7 @@ public function sortDescWorks() 'fieldName' => array('order' => 'desc') ) ); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertSame($expected, $actual['sort']); } @@ -213,7 +213,7 @@ public function rangeConstraintsWork($constraint, $operator, $value) 'fieldName' => array($operator => $value) ) ); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertInArray($expected, $actual['query']['filtered']['filter']['bool']['must']); } @@ -246,7 +246,7 @@ public function anSimpleAggregationCanBeAddedToTheRequest($type, $name, $field) ); $this->queryBuilder->fieldBasedAggregation($name, $field, $type); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertInArray($expected, $actual); } @@ -276,7 +276,7 @@ public function anAggregationCanBeSubbedUnderAPath() ) ); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertInArray($expected, $actual); } @@ -314,7 +314,7 @@ public function aCustomAggregationDefinitionCanBeApplied() ); $this->queryBuilder->aggregation("foo", $expected['foo']); - $actual = $this->queryBuilder->getRequest(); + $actual = $this->queryBuilder->getRequest()->toArray(); $this->assertInArray($expected, $actual); } From ebed605c0a3841310edbce3ecdc6c1853b7c12e5 Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Thu, 30 Jun 2016 18:20:42 +0200 Subject: [PATCH 02/41] TASK: Code style cleanup --- .../Domain/Model/AbstractQuery.php | 40 +++++++++++-------- .../Domain/Model/FilteredQuery.php | 25 +++++++----- .../Domain/Model/FunctionScoreQuery.php | 26 +++++++----- .../Domain/Model/QueryInterface.php | 18 ++++----- 4 files changed, 63 insertions(+), 46 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php index c4515463..0e9c5323 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php @@ -1,15 +1,15 @@ request['sort'] = []; } $this->request['sort'][] = $configuration; + return $this->queryBuilder; } @@ -124,6 +125,7 @@ protected function addSubAggregation($parentPath, $name, $aggregationConfigurati } $path[$name] = $aggregationConfiguration; + return $this->queryBuilder; } @@ -170,6 +172,7 @@ public function highlight($fragmentSize, $fragmentCount = null) public function setValueByPath($path, $value) { $this->request = Arrays::setValueByPath($this->request, $path, $value); + return $this->queryBuilder; } @@ -193,14 +196,16 @@ public function appendAtPath($path, array $data) /** * {@inheritdoc} */ - public function jsonSerialize() { + public function jsonSerialize() + { return $this->prepareRequest(); } /** * {@inheritdoc} */ - public function offsetSet($offset, $value) { + public function offsetSet($offset, $value) + { if (is_null($offset)) { $this->request[] = $value; } else { @@ -211,21 +216,24 @@ public function offsetSet($offset, $value) { /** * {@inheritdoc} */ - public function offsetExists($offset) { + public function offsetExists($offset) + { return isset($this->request[$offset]); } /** * {@inheritdoc} */ - public function offsetUnset($offset) { + public function offsetUnset($offset) + { unset($this->request[$offset]); } /** * {@inheritdoc} */ - public function offsetGet($offset) { + public function offsetGet($offset) + { return isset($this->request[$offset]) ? $this->request[$offset] : null; } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php index 7811ea61..85008738 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php @@ -1,15 +1,15 @@ request['size'] = (integer)$size; + return $this->queryBuilder; } @@ -91,6 +94,7 @@ public function size($size) public function from($size) { $this->request['from'] = (integer)$size; + return $this->queryBuilder; } @@ -104,6 +108,7 @@ public function fulltext($searchWord) 'query' => $searchWord ] ]); + return $this->queryBuilder; } @@ -115,7 +120,7 @@ public function queryFilter($filterType, $filterOptions, $clauseType = 'must') if (!in_array($clauseType, ['must', 'should', 'must_not'])) { throw new Exception\QueryBuildingException('The given clause type "' . $clauseType . '" is not supported. Must be one of "must", "should", "must_not".', 1383716082); } + return $this->appendAtPath('query.filtered.filter.bool.' . $clauseType, [$filterType => $filterOptions]); } - } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php index 637e608c..e4821f60 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FunctionScoreQuery.php @@ -1,15 +1,15 @@ functionScoreRequest['functions'] = $functions; } + return $this->queryBuilder; } @@ -49,6 +50,7 @@ public function scoreMode($scoreMode) throw new Exception\QueryBuildingException('Invalid score mode', 1454016230); } $this->functionScoreRequest['score_mode'] = $scoreMode; + return $this->queryBuilder; } @@ -63,6 +65,7 @@ public function boostMode($boostMode) throw new Exception\QueryBuildingException('Invalid boost mode', 1454016229); } $this->functionScoreRequest['boost_mode'] = $boostMode; + return $this->queryBuilder; } @@ -77,6 +80,7 @@ public function maxBoost($boost) throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); } $this->functionScoreRequest['max_boost'] = $boost; + return $this->queryBuilder; } @@ -91,6 +95,7 @@ public function minScore($score) throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); } $this->functionScoreRequest['min_score'] = $score; + return $this->queryBuilder; } @@ -114,8 +119,7 @@ protected function prepareRequest() 'function_score' => $functionScore ] ]); + return $query; } - - } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php index 8d03cf0d..7b623094 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/QueryInterface.php @@ -1,15 +1,15 @@ Date: Mon, 11 Jul 2016 11:42:36 +0200 Subject: [PATCH 03/41] TASK: Add Objects.yaml configuration to use FilteredQuery by default --- Configuration/Objects.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml index eeb4f0ba..6b24b19d 100644 --- a/Configuration/Objects.yaml +++ b/Configuration/Objects.yaml @@ -1,3 +1,6 @@ +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\QueryInterface: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\FilteredQuery' + Flowpack\ElasticSearch\ContentRepositoryAdaptor\Command\NodeIndexCommandController: properties: logger: @@ -34,4 +37,4 @@ Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface: Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient: scope: singleton factoryObjectName: Flowpack\ElasticSearch\ContentRepositoryAdaptor\Client\ClientFactory - factoryMethodName: create \ No newline at end of file + factoryMethodName: create From a07032203bec5ea264e67b3d171eeedc08f5b05e Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Mon, 11 Jul 2016 16:05:50 +0200 Subject: [PATCH 04/41] TASK: Fix broken unit tests --- Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php index 04775d43..3fc29194 100644 --- a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php +++ b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php @@ -11,6 +11,7 @@ * source code. */ +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\FilteredQuery; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\ElasticSearchQueryBuilder; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\TYPO3CR\Domain\Model\Workspace; @@ -41,7 +42,8 @@ public function setUp() $mockWorkspace->expects($this->any())->method('getName')->will($this->returnValue('user-foo')); $this->queryBuilder = new ElasticSearchQueryBuilder(); - $this->inject($this->queryBuilder, 'request', $request); + $query = new FilteredQuery($this->queryBuilder); + $this->inject($this->queryBuilder, 'request', $query); $this->queryBuilder->query($node); } From d8ddac16b382c640e13e13873295153f55302cce Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Tue, 31 Jan 2017 18:24:27 +0100 Subject: [PATCH 05/41] TASK: Extract query and driver for Elastic v1 --- .../Driver/AbstractDriver.php | 46 +++ .../Model => Driver}/AbstractQuery.php | 2 +- .../Driver/DriverInterface.php | 43 +++ .../Model => Driver}/QueryInterface.php | 2 +- .../Driver/Version1/Driver.php | 286 ++++++++++++++++++ .../Version1/Query}/FilteredQuery.php | 3 +- .../Version1/Query}/FunctionScoreQuery.php | 2 +- .../Eel/ElasticSearchQueryBuilder.php | 7 +- .../Indexer/NodeIndexer.php | 194 +----------- Configuration/Objects.yaml | 4 +- .../Eel/ElasticSearchQueryBuilderTest.php | 2 +- 11 files changed, 398 insertions(+), 193 deletions(-) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php rename Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/{Domain/Model => Driver}/AbstractQuery.php (99%) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DriverInterface.php rename Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/{Domain/Model => Driver}/QueryInterface.php (98%) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Driver.php rename Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/{Domain/Model => Driver/Version1/Query}/FilteredQuery.php (95%) rename Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/{Domain/Model => Driver/Version1/Query}/FunctionScoreQuery.php (97%) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php new file mode 100644 index 00000000..aefc14d1 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php @@ -0,0 +1,46 @@ +getNodeType()->hasConfiguration('search')) { + $elasticSearchSettingsForNode = $node->getNodeType()->getConfiguration('search'); + if (isset($elasticSearchSettingsForNode['fulltext']['isRoot']) && $elasticSearchSettingsForNode['fulltext']['isRoot'] === true) { + return true; + } + } + + return false; + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php similarity index 99% rename from Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php rename to Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php index 9a47329c..e1be6fbd 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/AbstractQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php @@ -1,5 +1,5 @@ searchClient->request('DELETE', '/' . implode(',', $indices) . '/'); + } + + /** + * @return array + */ + public function status() + { + return $this->searchClient->request('GET', '/_status')->getTreatedContent(); + } + + /** + * @param Index $index + * @param string|array $request + * @return array + */ + public function bulk(Index $index, $request) + { + if (is_array($request)) { + $request = json_encode($request); + } + + $response = $index->request('POST', '/_bulk', [], $request)->getOriginalResponse()->getContent(); + + return array_map(function ($line) { + return json_decode($line); + }, explode("\n", $response)); + } + + /** + * @param NodeInterface $node + * @param $identifier + * @return array + */ + public function delete(NodeInterface $node, $identifier) + { + return [ + [ + 'delete' => [ + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()), + '_id' => $identifier + ] + ] + ]; + } + + /** + * @param string $aliasName + * @return array + */ + public function currentlyLiveIndices($aliasName) + { + return array_keys($this->searchClient->request('GET', '/_alias/' . $aliasName)->getTreatedContent()); + } + + /** + * @param Index $index + * @param NodeInterface $node + * @param string $contextPathHash + */ + public function deleteByContextPathHash(Index $index, NodeInterface $node, $contextPathHash) + { + $index->request('DELETE', '/_query', [], json_encode([ + 'query' => [ + 'bool' => [ + 'must' => [ + 'ids' => [ + 'values' => [$contextPathHash] + ] + ], + 'must_not' => [ + 'term' => [ + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName()) + ] + ], + ] + ] + ])); + } + + /** + * @param array $actions + */ + public function aliasActions(array $actions) + { + $this->searchClient->request('POST', '/_aliases', [], \json_encode(['actions' => $actions])); + } + + /** + * @param string $aliasName + * @throws Exception + */ + public function removeAlias($aliasName) + { + $response = $this->searchClient->request('HEAD', '/' . $aliasName); + if ($response->getStatusCode() === 200) { + $response = $this->searchClient->request('DELETE', '/' . $aliasName); + if ($response->getStatusCode() !== 200) { + throw new Exception('The index "' . $aliasName . '" could not be removed to be replaced by an alias. (return code: ' . $response->getStatusCode() . ')', 1395419177); + } + } + } + + /** + * @param string $aliasName + * @return array + * @throws Exception + */ + public function indexNames($aliasName) + { + $response = $this->searchClient->request('GET', '/_alias/' . $aliasName); + if ($response->getStatusCode() !== 200) { + throw new Exception('The alias "' . $aliasName . '" was not found with some unexpected error... (return code: ' . $response->getStatusCode() . ')', 1383650137); + } + + return array_keys($response->getTreatedContent()); + } + + /** + * @param NodeInterface $node + * @param ElasticSearchDocument $document + * @param array $documentData + * @return array + */ + public function fulltextRootNode(NodeInterface $node, ElasticSearchDocument $document, array $documentData) + { + if ($this->isFulltextRoot($node)) { + // for fulltext root documents, we need to preserve the "__fulltext" field. That's why we use the + // "update" API instead of the "index" API, with a custom script internally; as we + // shall not delete the "__fulltext" part of the document if it has any. + return [ + [ + 'update' => [ + '_type' => $document->getType()->getName(), + '_id' => $document->getId() + ] + ], + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + [ + 'script' => ' + fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new LinkedHashMap()); + fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new LinkedHashMap()); + ctx._source = newData; + ctx._source.__fulltext = fulltext; + ctx._source.__fulltextParts = fulltextParts + ', + 'params' => [ + 'newData' => $documentData + ], + 'upsert' => $documentData, + 'lang' => 'groovy' + ] + ]; + } + + // non-fulltext-root documents can be indexed as-they-are + return [ + [ + 'index' => [ + '_type' => $document->getType()->getName(), + '_id' => $document->getId() + ] + ], + $documentData + ]; + } + + /** + * @param NodeInterface $node + * @param array $fulltextIndexOfNode + * @param string $targetWorkspaceName + * @return array + */ + public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targetWorkspaceName = null) + { + if ((($targetWorkspaceName !== null && $targetWorkspaceName !== 'live') || $node->getWorkspace()->getName() !== 'live') || count($fulltextIndexOfNode) === 0) { + return []; + } + + $closestFulltextNode = $node; + while (!$this->isFulltextRoot($closestFulltextNode)) { + $closestFulltextNode = $closestFulltextNode->getParent(); + if ($closestFulltextNode === null) { + // root of hierarchy, no fulltext root found anymore, abort silently... + $this->logger->log('No fulltext root found for ' . $node->getPath(), LOG_WARNING); + + return []; + } + } + + $closestFulltextNodeContextPath = str_replace($closestFulltextNode->getContext()->getWorkspace()->getName(), 'live', $closestFulltextNode->getContextPath()); + $closestFulltextNodeContextPathHash = sha1($closestFulltextNodeContextPath); + + return [ + [ + 'update' => [ + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()), + '_id' => $closestFulltextNodeContextPathHash + ] + ], + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + [ + // first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts + 'script' => ' + if (!ctx._source.containsKey("__fulltextParts")) { + ctx._source.__fulltextParts = new LinkedHashMap(); + } + ctx._source.__fulltextParts[identifier] = fulltext; + ctx._source.__fulltext = new LinkedHashMap(); + + Iterator> fulltextByNode = ctx._source.__fulltextParts.entrySet().iterator(); + for (fulltextByNode; fulltextByNode.hasNext();) { + Iterator> elementIterator = fulltextByNode.next().getValue().entrySet().iterator(); + for (elementIterator; elementIterator.hasNext();) { + Map.Entry element = elementIterator.next(); + String value; + + if (ctx._source.__fulltext.containsKey(element.key)) { + value = ctx._source.__fulltext[element.key] + " " + element.value.trim(); + } else { + value = element.value.trim(); + } + + ctx._source.__fulltext[element.key] = value; + } + } + ', + 'params' => [ + 'identifier' => $node->getIdentifier(), + 'fulltext' => $fulltextIndexOfNode + ], + 'upsert' => [ + '__fulltext' => $fulltextIndexOfNode, + '__fulltextParts' => [ + $node->getIdentifier() => $fulltextIndexOfNode + ] + ], + 'lang' => 'groovy' + ] + ]; + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php similarity index 95% rename from Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php rename to Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php index 85008738..b39fcaf5 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Domain/Model/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php @@ -1,5 +1,5 @@ bulkProcessing === false) { // Remove document with the same contextPathHash but different NodeType, required after NodeType change $this->logger->log(sprintf('NodeIndexer: Search and remove duplicate document if needed. ID: %s', $contextPath, $node->getNodeType()->getName(), $contextPathHash), LOG_DEBUG, null, 'ElasticSearch (CR)'); - $this->getIndex()->request('DELETE', '/_query', [], json_encode([ - 'query' => [ - 'bool' => [ - 'must' => [ - 'ids' => [ - 'values' => [$contextPathHash] - ] - ], - 'must_not' => [ - 'term' => [ - '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName()) - ] - ], - ] - ] - ])); + $this->driver->deleteByContextPathHash($this->getIndex(), $node, $contextPathHash); } if ($node->isRemoved()) { @@ -220,47 +205,8 @@ public function indexNode(NodeInterface $node, $targetWorkspaceName = null) } if ($this->isFulltextEnabled($node)) { - if ($this->isFulltextRoot($node)) { - // for fulltext root documents, we need to preserve the "__fulltext" field. That's why we use the - // "update" API instead of the "index" API, with a custom script internally; as we - // shall not delete the "__fulltext" part of the document if it has any. - $this->currentBulkRequest[] = [ - [ - 'update' => [ - '_type' => $document->getType()->getName(), - '_id' => $document->getId() - ] - ], - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html - [ - 'script' => ' - fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new LinkedHashMap()); - fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new LinkedHashMap()); - ctx._source = newData; - ctx._source.__fulltext = fulltext; - ctx._source.__fulltextParts = fulltextParts - ', - 'params' => [ - 'newData' => $documentData - ], - 'upsert' => $documentData, - 'lang' => 'groovy' - ] - ]; - } else { - // non-fulltext-root documents can be indexed as-they-are - $this->currentBulkRequest[] = [ - [ - 'index' => [ - '_type' => $document->getType()->getName(), - '_id' => $document->getId() - ] - ], - $documentData - ]; - } - - $this->updateFulltext($node, $fulltextIndexOfNode, $targetWorkspaceName); + $this->currentBulkRequest[] = $this->driver->fulltextRootNode($node, $document, $documentData); + $this->currentBulkRequest[] = $this->driver->fulltext($node, $fulltextIndexOfNode, $targetWorkspaceName); } $this->logger->log(sprintf('NodeIndexer: Added / updated node %s. ID: %s Context: %s', $contextPath, $contextPathHash, json_encode($node->getContext()->getProperties())), LOG_DEBUG, null, 'ElasticSearch (CR)'); @@ -286,101 +232,6 @@ public function indexNode(NodeInterface $node, $targetWorkspaceName = null) } } - /** - * - * - * @param NodeInterface $node - * @param array $fulltextIndexOfNode - * @param string $targetWorkspaceName - * @return void - */ - protected function updateFulltext(NodeInterface $node, array $fulltextIndexOfNode, $targetWorkspaceName = null) - { - if ((($targetWorkspaceName !== null && $targetWorkspaceName !== 'live') || $node->getWorkspace()->getName() !== 'live') || count($fulltextIndexOfNode) === 0) { - return; - } - - $closestFulltextNode = $node; - while (!$this->isFulltextRoot($closestFulltextNode)) { - $closestFulltextNode = $closestFulltextNode->getParent(); - if ($closestFulltextNode === null) { - // root of hierarchy, no fulltext root found anymore, abort silently... - $this->logger->log('No fulltext root found for ' . $node->getPath(), LOG_WARNING); - - return; - } - } - - $closestFulltextNodeContextPath = str_replace($closestFulltextNode->getContext()->getWorkspace()->getName(), 'live', $closestFulltextNode->getContextPath()); - $closestFulltextNodeContextPathHash = sha1($closestFulltextNodeContextPath); - - $this->currentBulkRequest[] = [ - [ - 'update' => [ - '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()), - '_id' => $closestFulltextNodeContextPathHash - ] - ], - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html - [ - // first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts - 'script' => ' - if (!ctx._source.containsKey("__fulltextParts")) { - ctx._source.__fulltextParts = new LinkedHashMap(); - } - ctx._source.__fulltextParts[identifier] = fulltext; - ctx._source.__fulltext = new LinkedHashMap(); - - Iterator> fulltextByNode = ctx._source.__fulltextParts.entrySet().iterator(); - for (fulltextByNode; fulltextByNode.hasNext();) { - Iterator> elementIterator = fulltextByNode.next().getValue().entrySet().iterator(); - for (elementIterator; elementIterator.hasNext();) { - Map.Entry element = elementIterator.next(); - String value; - - if (ctx._source.__fulltext.containsKey(element.key)) { - value = ctx._source.__fulltext[element.key] + " " + element.value.trim(); - } else { - value = element.value.trim(); - } - - ctx._source.__fulltext[element.key] = value; - } - } - ', - 'params' => [ - 'identifier' => $node->getIdentifier(), - 'fulltext' => $fulltextIndexOfNode - ], - 'upsert' => [ - '__fulltext' => $fulltextIndexOfNode, - '__fulltextParts' => [ - $node->getIdentifier() => $fulltextIndexOfNode - ] - ], - 'lang' => 'groovy' - ] - ]; - } - - /** - * Whether the node is configured as fulltext root. - * - * @param NodeInterface $node - * @return boolean - */ - protected function isFulltextRoot(NodeInterface $node) - { - if ($node->getNodeType()->hasConfiguration('search')) { - $elasticSearchSettingsForNode = $node->getNodeType()->getConfiguration('search'); - if (isset($elasticSearchSettingsForNode['fulltext']['isRoot']) && $elasticSearchSettingsForNode['fulltext']['isRoot'] === true) { - return true; - } - } - - return false; - } - /** * Schedule node removal into the current bulk request. * @@ -398,14 +249,7 @@ public function removeNode(NodeInterface $node) // TODO: handle deletion from the fulltext index as well $identifier = sha1($node->getContextPath()); - $this->currentBulkRequest[] = [ - [ - 'delete' => [ - '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()), - '_id' => $identifier - ] - ] - ]; + $this->currentBulkRequest[] = $this->driver->delete($node, $identifier); $this->logger->log(sprintf('NodeIndexer: Removed node %s from index (node actually removed). Persistence ID: %s', $node->getContextPath(), $identifier), LOG_DEBUG, null, 'ElasticSearch (CR)'); } @@ -436,9 +280,8 @@ public function flush() } if ($content !== '') { - $responseAsLines = $this->getIndex()->request('POST', '/_bulk', [], $content)->getOriginalResponse()->getContent(); - foreach (explode("\n", $responseAsLines) as $responseLine) { - $response = json_decode($responseLine); + $response = $this->driver->bulk($this->getIndex(), $content); + foreach ($response as $responseLine) { if (!is_object($response) || (isset($response->errors) && $response->errors !== false)) { $this->logger->log('Indexing Error: ' . $responseLine, LOG_ERR); } @@ -469,22 +312,11 @@ public function updateIndexAlias() $aliasActions = []; try { - $response = $this->searchClient->request('GET', '/_alias/' . $aliasName); - if ($response->getStatusCode() !== 200) { - throw new Exception('The alias "' . $aliasName . '" was not found with some unexpected error... (return code: ' . $response->getStatusCode() . ')', 1383650137); - } - - $indexNames = array_keys($response->getTreatedContent()); + $indexNames = $this->driver->indexNames($aliasName); if ($indexNames === []) { // if there is an actual index with the name we want to use as alias, remove it now - $response = $this->searchClient->request('HEAD', '/' . $aliasName); - if ($response->getStatusCode() === 200) { - $response = $this->searchClient->request('DELETE', '/' . $aliasName); - if ($response->getStatusCode() !== 200) { - throw new Exception('The index "' . $aliasName . '" could not be removed to be replaced by an alias. (return code: ' . $response->getStatusCode() . ')', 1395419177); - } - } + $this->driver->removeAlias($aliasName); } else { foreach ($indexNames as $indexName) { $aliasActions[] = [ @@ -509,7 +341,7 @@ public function updateIndexAlias() ] ]; - $this->searchClient->request('POST', '/_aliases', [], \json_encode(['actions' => $aliasActions])); + $this->driver->aliasActions($aliasActions); } /** @@ -522,9 +354,9 @@ public function removeOldIndices() { $aliasName = $this->searchClient->getIndexName(); // The alias name is the unprefixed index name - $currentlyLiveIndices = array_keys($this->searchClient->request('GET', '/_alias/' . $aliasName)->getTreatedContent()); + $currentlyLiveIndices = $this->driver->currentlyLiveIndices($aliasName); - $indexStatus = $this->searchClient->request('GET', '/_status')->getTreatedContent(); + $indexStatus = $this->driver->status(); $allIndices = array_keys($indexStatus['indices']); $indicesToBeRemoved = []; @@ -543,9 +375,7 @@ public function removeOldIndices() $indicesToBeRemoved[] = $indexName; } - if (count($indicesToBeRemoved) > 0) { - $this->searchClient->request('DELETE', '/' . implode(',', $indicesToBeRemoved) . '/'); - } + $this->driver->deleteIndices($indicesToBeRemoved); return $indicesToBeRemoved; } diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml index 6b24b19d..56b79f21 100644 --- a/Configuration/Objects.yaml +++ b/Configuration/Objects.yaml @@ -1,5 +1,5 @@ -Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\QueryInterface: - className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\FilteredQuery' +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Query\FilteredQuery' Flowpack\ElasticSearch\ContentRepositoryAdaptor\Command\NodeIndexCommandController: properties: diff --git a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php index 0d068b3a..61d2de14 100644 --- a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php +++ b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php @@ -11,7 +11,7 @@ * source code. */ -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Domain\Model\FilteredQuery; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Query\FilteredQuery; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\ElasticSearchQueryBuilder; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\TYPO3CR\Domain\Model\Workspace; From 0f11688cd098fc5d42e3fba8df42dcb4ee5795b4 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Tue, 31 Jan 2017 18:53:19 +0100 Subject: [PATCH 06/41] TASK: Cleanup NodeIndexer dependencies --- .../Indexer/NodeIndexer.php | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php index c65f6a28..a5a52ff8 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php @@ -22,7 +22,6 @@ use TYPO3\TYPO3CR\Domain\Service\ContentDimensionCombinator; use TYPO3\TYPO3CR\Domain\Service\Context; use TYPO3\TYPO3CR\Domain\Service\ContextFactory; -use TYPO3\TYPO3CR\Domain\Service\NodeTypeManager; use TYPO3\TYPO3CR\Search\Indexer\AbstractNodeIndexer; use TYPO3\TYPO3CR\Search\Indexer\BulkNodeIndexerInterface; @@ -48,24 +47,6 @@ class NodeIndexer extends AbstractNodeIndexer implements BulkNodeIndexerInterfac */ protected $searchClient; - /** - * @Flow\Inject - * @var NodeTypeMappingBuilder - */ - protected $nodeTypeMappingBuilder; - - /** - * @Flow\Inject - * @var \TYPO3\Flow\Persistence\PersistenceManagerInterface - */ - protected $persistenceManager; - - /** - * @Flow\Inject - * @var NodeTypeManager - */ - protected $nodeTypeManager; - /** * @Flow\Inject * @var \Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface From f6a04038a1b7d569de43f8e669e328c47d679f5f Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 16:44:02 +0100 Subject: [PATCH 07/41] TASK: Better Interfaces and create dedicated drivers for Elastic 1.x and 2.x --- .../Driver/AbstractDriver.php | 51 ++-- .../Driver/AbstractQuery.php | 40 +-- .../Driver/DocumentDriverInterface.php | 39 +++ .../Driver/DriverInterface.php | 41 --- .../Driver/IndexManagementDriverInterface.php | 50 ++++ .../Driver/IndexerDriverInterface.php | 41 +++ .../Driver/QueryInterface.php | 16 +- .../Driver/RequestDriverInterface.php | 28 ++ .../Driver/SystemDriverInterface.php | 25 ++ .../Driver/Version1/DocumentDriver.php | 66 +++++ .../Driver/Version1/Driver.php | 253 ------------------ .../Driver/Version1/IndexManagementDriver.php | 72 +++++ .../Driver/Version1/IndexerDriver.php | 162 +++++++++++ .../Driver/Version1/Query/FilteredQuery.php | 6 - .../Version1/Query/FunctionScoreQuery.php | 125 --------- .../Driver/Version1/RequestDriver.php | 41 +++ .../Driver/Version1/SystemDriver.php | 32 +++ .../Driver/Version2/DocumentDriver.php | 79 ++++++ .../Driver/Version2/IndexManagementDriver.php | 24 ++ .../Driver/Version2/IndexerDriver.php | 161 +++++++++++ .../Driver/Version2/Query/FilteredQuery.php | 121 +++++++++ .../Driver/Version2/RequestDriver.php | 24 ++ .../Driver/Version2/SystemDriver.php | 25 ++ .../Eel/ElasticSearchQueryBuilder.php | 19 +- .../DriverConfigurationException.php | 22 ++ .../Exception/QueryBuildingException.php | 3 +- .../Factory/DriverFactory.php | 120 +++++++++ .../Indexer/NodeIndexer.php | 76 ++++-- Configuration/Objects.yaml | 31 ++- Configuration/Settings.yaml | 23 +- 30 files changed, 1286 insertions(+), 530 deletions(-) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php delete mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DriverInterface.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexerDriverInterface.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/RequestDriverInterface.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/SystemDriverInterface.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php delete mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Driver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php delete mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php index aefc14d1..51bfe34e 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php @@ -11,36 +11,43 @@ * source code. */ +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; -use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\Flow\Annotations as Flow; +use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** * Abstract Fulltext Indexer */ abstract class AbstractDriver { - /** - * @Flow\Inject - * @var LoggerInterface - */ - protected $logger; + /** + * @Flow\Inject + * @var ElasticSearchClient + */ + protected $searchClient; + + /** + * @Flow\Inject + * @var LoggerInterface + */ + protected $logger; - /** - * Whether the node is configured as fulltext root. - * - * @param NodeInterface $node - * @return boolean - */ - protected function isFulltextRoot(NodeInterface $node) - { - if ($node->getNodeType()->hasConfiguration('search')) { - $elasticSearchSettingsForNode = $node->getNodeType()->getConfiguration('search'); - if (isset($elasticSearchSettingsForNode['fulltext']['isRoot']) && $elasticSearchSettingsForNode['fulltext']['isRoot'] === true) { - return true; - } - } + /** + * Whether the node is configured as fulltext root. + * + * @param NodeInterface $node + * @return boolean + */ + protected function isFulltextRoot(NodeInterface $node) + { + if ($node->getNodeType()->hasConfiguration('search')) { + $elasticSearchSettingsForNode = $node->getNodeType()->getConfiguration('search'); + if (isset($elasticSearchSettingsForNode['fulltext']['isRoot']) && $elasticSearchSettingsForNode['fulltext']['isRoot'] === true) { + return true; + } + } - return false; - } + return false; + } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php index e1be6fbd..15959a31 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php @@ -14,7 +14,6 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; use TYPO3\Eel\ProtectedContextAwareInterface; use TYPO3\Flow\Utility\Arrays; -use TYPO3\TYPO3CR\Search\Search\QueryBuilderInterface; /** * Default Filtered Query @@ -28,11 +27,6 @@ abstract class AbstractQuery implements QueryInterface, \JsonSerializable, \Arra */ protected $request = []; - /** - * @var QueryBuilderInterface - */ - protected $queryBuilder; - /** * These fields are not accepted in a count request and must therefore be removed before doing so * @@ -40,18 +34,6 @@ abstract class AbstractQuery implements QueryInterface, \JsonSerializable, \Arra */ protected $unsupportedFieldsInCountRequest = ['fields', 'sort', 'from', 'size', 'highlight', 'aggs', 'aggregations']; - /** - * @param QueryBuilderInterface $queryBuilder - * @param array $request Override the default request - */ - public function __construct(QueryBuilderInterface $queryBuilder, array $request = null) - { - $this->queryBuilder = $queryBuilder; - if ($request !== null) { - $this->request = $request; - } - } - /** * Modify a part of the Elasticsearch Request denoted by $path, merging together * the existing values and the passed-in values. @@ -99,8 +81,6 @@ public function addSortFilter($configuration) $this->request['sort'] = []; } $this->request['sort'][] = $configuration; - - return $this->queryBuilder; } /** @@ -117,8 +97,6 @@ public function aggregation($name, array $aggregationDefinition, $parentPath = n } else { $this->request['aggregations'][$name] = $aggregationDefinition; } - - return $this->queryBuilder; } /** @@ -147,8 +125,6 @@ protected function addSubAggregation($parentPath, $name, $aggregationConfigurati } $path[$name] = $aggregationConfiguration; - - return $this->queryBuilder; } /** @@ -161,8 +137,6 @@ public function suggestions($name, array $suggestionDefinition) } $this->request['suggest'][$name] = $suggestionDefinition; - - return $this->queryBuilder; } /** @@ -184,8 +158,6 @@ public function highlight($fragmentSize, $fragmentCount = null) ] ]; } - - return $this->queryBuilder; } /** @@ -194,8 +166,6 @@ public function highlight($fragmentSize, $fragmentCount = null) public function setValueByPath($path, $value) { $this->request = Arrays::setValueByPath($this->request, $path, $value); - - return $this->queryBuilder; } /** @@ -211,8 +181,6 @@ public function appendAtPath($path, array $data) $currentElement =& $currentElement[$pathPart]; } $currentElement[] = $data; - - return $this->queryBuilder; } /** @@ -281,4 +249,12 @@ public function allowsCallOfMethod($methodName) { return true; } + + /** + * @param array $request + */ + public function replaceRequest(array $request) + { + $this->request = $request; + } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php new file mode 100644 index 00000000..15b8cc0b --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php @@ -0,0 +1,39 @@ + [ + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()), + '_id' => $identifier + ] + ] + ]; + } + + /** + * {@inheritdoc} + */ + public function deleteByDocumentIdentifier(Index $index, NodeInterface $node, $documentIdentifier) + { + $index->request('DELETE', '/_query', [], json_encode([ + 'query' => [ + 'bool' => [ + 'must' => [ + 'ids' => [ + 'values' => [$documentIdentifier] + ] + ], + 'must_not' => [ + 'term' => [ + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName()) + ] + ], + ] + ] + ])); + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Driver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Driver.php deleted file mode 100644 index c00ae3fa..00000000 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Driver.php +++ /dev/null @@ -1,253 +0,0 @@ -searchClient->request('DELETE', '/' . implode(',', $indices) . '/'); - } - - /** - * @return array - */ - public function status() - { - return $this->searchClient->request('GET', '/_status')->getTreatedContent(); - } - - /** - * @param Index $index - * @param string|array $request - * @return array - */ - public function bulk(Index $index, $request) - { - if (is_array($request)) { - $request = json_encode($request); - } - - $response = $index->request('POST', '/_bulk', [], $request)->getOriginalResponse()->getContent(); - - return array_map(function ($line) { - return json_decode($line); - }, explode("\n", $response)); - } - - /** - * @param NodeInterface $node - * @param $identifier - * @return array - */ - public function delete(NodeInterface $node, $identifier) - { - return [ - [ - 'delete' => [ - '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()), - '_id' => $identifier - ] - ] - ]; - } - - /** - * @param string $aliasName - * @return array - */ - public function currentlyLiveIndices($aliasName) - { - return array_keys($this->searchClient->request('GET', '/_alias/' . $aliasName)->getTreatedContent()); - } - - /** - * @param Index $index - * @param NodeInterface $node - * @param string $documentIdentifier - */ - public function deleteByDocumentIdentifier(Index $index, NodeInterface $node, $documentIdentifier) - { - $index->request('DELETE', '/_query', [], json_encode([ - 'query' => [ - 'bool' => [ - 'must' => [ - 'ids' => [ - 'values' => [$documentIdentifier] - ] - ], - 'must_not' => [ - 'term' => [ - '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName()) - ] - ], - ] - ] - ])); - } - - /** - * @param array $actions - */ - public function aliasActions(array $actions) - { - $this->searchClient->request('POST', '/_aliases', [], \json_encode(['actions' => $actions])); - } - - /** - * @param string $aliasName - * @throws Exception - */ - public function removeAlias($aliasName) - { - $response = $this->searchClient->request('HEAD', '/' . $aliasName); - if ($response->getStatusCode() === 200) { - $response = $this->searchClient->request('DELETE', '/' . $aliasName); - if ($response->getStatusCode() !== 200) { - throw new Exception('The index "' . $aliasName . '" could not be removed to be replaced by an alias. (return code: ' . $response->getStatusCode() . ')', 1395419177); - } - } - } - - /** - * @param string $aliasName - * @return array - * @throws Exception - */ - public function indexNames($aliasName) - { - $response = $this->searchClient->request('GET', '/_alias/' . $aliasName); - if ($response->getStatusCode() !== 200) { - throw new Exception('The alias "' . $aliasName . '" was not found with some unexpected error... (return code: ' . $response->getStatusCode() . ')', 1383650137); - } - - return array_keys($response->getTreatedContent()); - } - - /** - * @param NodeInterface $node - * @param array $fulltextIndexOfNode - * @param string $targetWorkspaceName - * @return array - */ - public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targetWorkspaceName = null) - { - $closestFulltextNode = $node; - while (!$this->isFulltextRoot($closestFulltextNode)) { - $closestFulltextNode = $closestFulltextNode->getParent(); - if ($closestFulltextNode === null) { - // root of hierarchy, no fulltext root found anymore, abort silently... - $this->logger->log(sprintf('NodeIndexer: No fulltext root found for node %s (%)', $node->getPath(), $node->getIdentifier()), LOG_WARNING, null, 'ElasticSearch (CR)'); - - return null; - } - } - - $closestFulltextNodeContextPath = $closestFulltextNode->getContextPath(); - if ($targetWorkspaceName !== null) { - $closestFulltextNodeContextPath = str_replace($node->getContext()->getWorkspace()->getName(), $targetWorkspaceName, $closestFulltextNodeContextPath); - } - $closestFulltextNodeDocumentIdentifier = sha1($closestFulltextNodeContextPath); - - if ($closestFulltextNode->isRemoved()) { - // fulltext root is removed, abort silently... - $this->logger->log(sprintf('NodeIndexer (%s): Fulltext root found for %s (%s) not updated, it is removed', $closestFulltextNodeDocumentIdentifier, $node->getPath(), $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); - return null; - } - - $this->logger->log(sprintf('NodeIndexer (%s): Updated fulltext index for %s (%s)', $closestFulltextNodeDocumentIdentifier, $closestFulltextNodeContextPath, $closestFulltextNode->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); - - return [ - [ - 'update' => [ - '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()), - '_id' => $closestFulltextNodeDocumentIdentifier - ] - ], - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html - [ - // first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts - 'script' => ' - if (!ctx._source.containsKey("__fulltextParts")) { - ctx._source.__fulltextParts = new LinkedHashMap(); - } - - if (nodeIsRemoved || nodeIsHidden || fulltext.size() == 0) { - if (ctx._source.__fulltextParts.containsKey(identifier)) { - ctx._source.__fulltextParts.remove(identifier); - } - } else { - ctx._source.__fulltextParts[identifier] = fulltext; - } - ctx._source.__fulltext = new LinkedHashMap(); - - Iterator> fulltextByNode = ctx._source.__fulltextParts.entrySet().iterator(); - for (fulltextByNode; fulltextByNode.hasNext();) { - Iterator> elementIterator = fulltextByNode.next().getValue().entrySet().iterator(); - for (elementIterator; elementIterator.hasNext();) { - Map.Entry element = elementIterator.next(); - String value; - - if (ctx._source.__fulltext.containsKey(element.key)) { - value = ctx._source.__fulltext[element.key] + " " + element.value.trim(); - } else { - value = element.value.trim(); - } - - ctx._source.__fulltext[element.key] = value; - } - } - ', - 'params' => [ - 'identifier' => $node->getIdentifier(), - 'nodeIsRemoved' => $node->isRemoved(), - 'nodeIsHidden' => $node->isHidden(), - 'fulltext' => $fulltextIndexOfNode - ], - 'upsert' => [ - '__fulltext' => $fulltextIndexOfNode, - '__fulltextParts' => [ - $node->getIdentifier() => $fulltextIndexOfNode - ] - ], - 'lang' => 'groovy' - ] - ]; - } -} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php new file mode 100644 index 00000000..2add9fc2 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php @@ -0,0 +1,72 @@ +searchClient->request('DELETE', '/' . implode(',', $indices) . '/'); + } + + /** + * {@inheritdoc} + */ + public function actions(array $actions) + { + $this->searchClient->request('POST', '/_aliases', [], \json_encode(['actions' => $actions])); + } + + /** + * {@inheritdoc} + */ + public function remove($aliasName) + { + $response = $this->searchClient->request('HEAD', '/' . $aliasName); + if ($response->getStatusCode() === 200) { + $response = $this->searchClient->request('DELETE', '/' . $aliasName); + if ($response->getStatusCode() !== 200) { + throw new Exception('The index "' . $aliasName . '" could not be removed to be replaced by an alias. (return code: ' . $response->getStatusCode() . ')', 1395419177); + } + } + } + + /** + * {@inheritdoc} + */ + public function indexesByAlias($alias) + { + $response = $this->searchClient->request('GET', '/_alias/' . $alias); + if ($response->getStatusCode() !== 200) { + throw new Exception('The alias "' . $alias . '" was not found with some unexpected error... (return code: ' . $response->getStatusCode() . ')', 1383650137); + } + + return array_keys($response->getTreatedContent()); + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php new file mode 100644 index 00000000..c15322b4 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php @@ -0,0 +1,162 @@ +isFulltextRoot($node)) { + // for fulltext root documents, we need to preserve the "__fulltext" field. That's why we use the + // "update" API instead of the "index" API, with a custom script internally; as we + // shall not delete the "__fulltext" part of the document if it has any. + return [ + [ + 'update' => [ + '_type' => $document->getType()->getName(), + '_id' => $document->getId() + ] + ], + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + [ + 'script' => ' + fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new LinkedHashMap()); + fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new LinkedHashMap()); + ctx._source = newData; + ctx._source.__fulltext = fulltext; + ctx._source.__fulltextParts = fulltextParts + ', + 'params' => [ + 'newData' => $documentData + ], + 'upsert' => $documentData, + 'lang' => 'groovy' + ] + ]; + } else { + // non-fulltext-root documents can be indexed as-they-are + return [ + [ + 'index' => [ + '_type' => $document->getType()->getName(), + '_id' => $document->getId() + ] + ], + $documentData + ]; + } + } + + /** + * {@inheritdoc} + */ + public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targetWorkspaceName = null) + { + $closestFulltextNode = $node; + while (!$this->isFulltextRoot($closestFulltextNode)) { + $closestFulltextNode = $closestFulltextNode->getParent(); + if ($closestFulltextNode === null) { + // root of hierarchy, no fulltext root found anymore, abort silently... + $this->logger->log(sprintf('NodeIndexer: No fulltext root found for node %s (%)', $node->getPath(), $node->getIdentifier()), LOG_WARNING, null, 'ElasticSearch (CR)'); + + return null; + } + } + + $closestFulltextNodeContextPath = $closestFulltextNode->getContextPath(); + if ($targetWorkspaceName !== null) { + $closestFulltextNodeContextPath = str_replace($node->getContext()->getWorkspace()->getName(), $targetWorkspaceName, $closestFulltextNodeContextPath); + } + $closestFulltextNodeDocumentIdentifier = sha1($closestFulltextNodeContextPath); + + if ($closestFulltextNode->isRemoved()) { + // fulltext root is removed, abort silently... + $this->logger->log(sprintf('NodeIndexer (%s): Fulltext root found for %s (%s) not updated, it is removed', $closestFulltextNodeDocumentIdentifier, $node->getPath(), $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); + return null; + } + + $this->logger->log(sprintf('NodeIndexer (%s): Updated fulltext index for %s (%s)', $closestFulltextNodeDocumentIdentifier, $closestFulltextNodeContextPath, $closestFulltextNode->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); + + return [ + [ + 'update' => [ + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()), + '_id' => $closestFulltextNodeDocumentIdentifier + ] + ], + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + [ + // first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts + 'script' => ' + if (!ctx._source.containsKey("__fulltextParts")) { + ctx._source.__fulltextParts = new LinkedHashMap(); + } + + if (nodeIsRemoved || nodeIsHidden || fulltext.size() == 0) { + if (ctx._source.__fulltextParts.containsKey(identifier)) { + ctx._source.__fulltextParts.remove(identifier); + } + } else { + ctx._source.__fulltextParts[identifier] = fulltext; + } + ctx._source.__fulltext = new LinkedHashMap(); + + Iterator> fulltextByNode = ctx._source.__fulltextParts.entrySet().iterator(); + for (fulltextByNode; fulltextByNode.hasNext();) { + Iterator> elementIterator = fulltextByNode.next().getValue().entrySet().iterator(); + for (elementIterator; elementIterator.hasNext();) { + Map.Entry element = elementIterator.next(); + String value; + + if (ctx._source.__fulltext.containsKey(element.key)) { + value = ctx._source.__fulltext[element.key] + " " + element.value.trim(); + } else { + value = element.value.trim(); + } + + ctx._source.__fulltext[element.key] = value; + } + } + ', + 'params' => [ + 'identifier' => $node->getIdentifier(), + 'nodeIsRemoved' => $node->isRemoved(), + 'nodeIsHidden' => $node->isHidden(), + 'fulltext' => $fulltextIndexOfNode + ], + 'upsert' => [ + '__fulltext' => $fulltextIndexOfNode, + '__fulltextParts' => [ + $node->getIdentifier() => $fulltextIndexOfNode + ] + ], + 'lang' => 'groovy' + ] + ]; + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php index b39fcaf5..369afa99 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php @@ -85,8 +85,6 @@ public function getCountRequestAsJSON() public function size($size) { $this->request['size'] = (integer)$size; - - return $this->queryBuilder; } /** @@ -95,8 +93,6 @@ public function size($size) public function from($size) { $this->request['from'] = (integer)$size; - - return $this->queryBuilder; } /** @@ -109,8 +105,6 @@ public function fulltext($searchWord) 'query' => $searchWord ] ]); - - return $this->queryBuilder; } /** diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php deleted file mode 100644 index 8f321ea0..00000000 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php +++ /dev/null @@ -1,125 +0,0 @@ - [] - ]; - - /** - * @param array $functions - * @return QueryBuilderInterface - */ - public function functions(array $functions) - { - if (isset($functions['functions'])) { - $this->functionScoreRequest = $functions; - } else { - $this->functionScoreRequest['functions'] = $functions; - } - - return $this->queryBuilder; - } - - /** - * @param string $scoreMode - * @return QueryBuilderInterface - * @throws Exception\QueryBuildingException - */ - public function scoreMode($scoreMode) - { - if (!in_array($scoreMode, ['multiply', 'first', 'sum', 'avg', 'max', 'min'])) { - throw new Exception\QueryBuildingException('Invalid score mode', 1454016230); - } - $this->functionScoreRequest['score_mode'] = $scoreMode; - - return $this->queryBuilder; - } - - /** - * @param string $boostMode - * @return QueryBuilderInterface - * @throws Exception\QueryBuildingException - */ - public function boostMode($boostMode) - { - if (!in_array($boostMode, ['multiply', 'replace', 'sum', 'avg', 'max', 'min'])) { - throw new Exception\QueryBuildingException('Invalid boost mode', 1454016229); - } - $this->functionScoreRequest['boost_mode'] = $boostMode; - - return $this->queryBuilder; - } - - /** - * @param integer|float $boost - * @return QueryBuilderInterface - * @throws Exception\QueryBuildingException - */ - public function maxBoost($boost) - { - if (!is_numeric($boost)) { - throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); - } - $this->functionScoreRequest['max_boost'] = $boost; - - return $this->queryBuilder; - } - - /** - * @param integer|float $score - * @return QueryBuilderInterface - * @throws Exception\QueryBuildingException - */ - public function minScore($score) - { - if (!is_numeric($score)) { - throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); - } - $this->functionScoreRequest['min_score'] = $score; - - return $this->queryBuilder; - } - - /** - * {@inheritdoc} - */ - protected function prepareRequest() - { - if ($this->functionScoreRequest['functions'] === []) { - return parent::prepareRequest(); - } - $currentQuery = $this->request['query']; - - $baseQuery = $this->request; - unset($baseQuery['query']); - - $functionScore = $this->functionScoreRequest; - $functionScore['query'] = $currentQuery; - $query = Arrays::arrayMergeRecursiveOverrule($baseQuery, [ - 'query' => [ - 'function_score' => $functionScore - ] - ]); - - return $query; - } -} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php new file mode 100644 index 00000000..0155947d --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php @@ -0,0 +1,41 @@ +request('POST', '/_bulk', [], $request)->getOriginalResponse()->getContent(); + + return array_map(function ($line) { + return json_decode($line); + }, explode("\n", $response)); + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php new file mode 100644 index 00000000..05f1e959 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php @@ -0,0 +1,32 @@ +searchClient->request('GET', '/_status')->getTreatedContent(); + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php new file mode 100644 index 00000000..2ad66d5e --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php @@ -0,0 +1,79 @@ +getNodeType()->getName()); + $result = $index->request('GET', '/_search?scroll=1m', [], json_encode([ + 'sort' => ['_doc'], + 'query' => [ + 'bool' => [ + 'must' => [ + 'ids' => [ + 'values' => [$documentIdentifier] + ] + ], + 'must_not' => [ + 'term' => [ + '_type' => $type + ] + ] + ] + ] + ])); + $treatedContent = $result->getTreatedContent(); + $scrollId = $treatedContent['_scroll_id']; + $mapHitToDeleteRequest = function ($hit) { + $bulkRequest[] = json_encode([ + 'delete' => [ + '_type' => $hit['_type'], + '_id' => $hit['_id'] + ] + ]); + }; + $bulkRequest = []; + while (isset($treatedContent['hits']['hits']) && $treatedContent['hits']['hits'] !== []) { + $hits = $treatedContent['hits']['hits']; + $bulkRequest = array_merge($bulkRequest, array_map($mapHitToDeleteRequest, $hits)); + $result = $index->request('GET', '/_search/scroll?scroll=1m', [], $scrollId, false); + $treatedContent = $result->getTreatedContent(); + } + $this->logger->log(sprintf('NodeIndexer: Check duplicate nodes for %s (%s), found %d document(s)', $documentIdentifier, $type, count($bulkRequest)), LOG_DEBUG, null, 'ElasticSearch (CR)'); + if ($bulkRequest !== []) { + $index->request('POST', '/_bulk', [], implode("\n", $bulkRequest)); + } + $this->searchClient->request('DELETE', '/_search/scroll', [], json_encode([ + 'scroll_id' => [ + $scrollId + ] + ])); + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php new file mode 100644 index 00000000..ae783e8b --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php @@ -0,0 +1,24 @@ +isFulltextRoot($node)) { + // for fulltext root documents, we need to preserve the "__fulltext" field. That's why we use the + // "update" API instead of the "index" API, with a custom script internally; as we + // shall not delete the "__fulltext" part of the document if it has any. + return [ + [ + 'update' => [ + '_type' => $document->getType()->getName(), + '_id' => $document->getId(), + '_index' => $indexName, + '_retry_on_conflict' => 3 + ] + ], + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + [ + 'script' => [ + 'inline' => ' + fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new HashMap()); + fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new HashMap()); + ctx._source = newData; + ctx._source.__fulltext = fulltext; + ctx._source.__fulltextParts = fulltextParts + ', + 'params' => [ + 'newData' => $documentData + ] + ], + 'upsert' => $documentData, + 'lang' => 'groovy' + ] + ]; + } + + // non-fulltext-root documents can be indexed as-they-are + return [ + [ + 'index' => [ + '_type' => $document->getType()->getName(), + '_id' => $document->getId() + ] + ], + $documentData + ]; + } + + /** + * {@inheritdoc} + */ + public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targetWorkspaceName = null) + { + $closestFulltextNode = $node; + while (!$this->isFulltextRoot($closestFulltextNode)) { + $closestFulltextNode = $closestFulltextNode->getParent(); + if ($closestFulltextNode === null) { + // root of hierarchy, no fulltext root found anymore, abort silently... + $this->logger->log(sprintf('NodeIndexer: No fulltext root found for node %s (%)', $node->getPath(), $node->getIdentifier()), LOG_WARNING, null, 'ElasticSearch (CR)'); + + return null; + } + } + + $closestFulltextNodeContextPath = $closestFulltextNode->getContextPath(); + if ($targetWorkspaceName !== null) { + $closestFulltextNodeContextPath = str_replace($node->getContext()->getWorkspace()->getName(), $targetWorkspaceName, $closestFulltextNodeContextPath); + } + $closestFulltextNodeDocumentIdentifier = sha1($closestFulltextNodeContextPath); + + if ($closestFulltextNode->isRemoved()) { + // fulltext root is removed, abort silently... + $this->logger->log(sprintf('NodeIndexer (%s): Fulltext root found for %s (%s) not updated, it is removed', $closestFulltextNodeDocumentIdentifier, $node->getPath(), $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); + return null; + } + + $this->logger->log(sprintf('NodeIndexer (%s): Updated fulltext index for %s (%s)', $closestFulltextNodeDocumentIdentifier, $closestFulltextNodeContextPath, $closestFulltextNode->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); + + $upsertFulltextParts = []; + if (!empty($fulltextIndexOfNode)) { + $upsertFulltextParts[$node->getIdentifier()] = $fulltextIndexOfNode; + } + + return [ + [ + 'update' => [ + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()), + '_id' => $closestFulltextNodeDocumentIdentifier + ] + ], + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + [ + // first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts + 'script' => [ + 'inline' => ' + ctx._source.__fulltext = new HashMap(); + + if (!ctx._source.containsKey("__fulltextParts")) { + ctx._source.__fulltextParts = new HashMap(); + } + + if (fulltext.size() == 0) { + ctx._source.__fulltextParts.remove(identifier); + } else { + ctx._source.__fulltextParts.put(identifier, fulltext); + } + + ctx._source.__fulltextParts.each { originNodeIdentifier, partContent -> partContent.each { bucketKey, content -> + if (ctx._source.__fulltext.containsKey(bucketKey)) { + value = ctx._source.__fulltext[bucketKey] + " " + content.trim(); + } else { + value = content.trim(); + } + ctx._source.__fulltext[bucketKey] = value; + } + } + ', + 'params' => [ + 'identifier' => $node->getIdentifier(), + 'nodeIsRemoved' => $node->isRemoved(), + 'nodeIsHidden' => $node->isHidden(), + 'fulltext' => $fulltextIndexOfNode + ], + ], + 'upsert' => [ + '__fulltext' => $fulltextIndexOfNode, + '__fulltextParts' => $upsertFulltextParts + ], + 'lang' => 'groovy' + ] + ]; + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php new file mode 100644 index 00000000..7bb9a51b --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php @@ -0,0 +1,121 @@ + [ + 'filtered' => [ + 'query' => [ + 'bool' => [ + 'must' => [ + [ + 'match_all' => [] + ] + ] + ] + + ], + 'filter' => [ + 'bool' => [ + 'must' => [], + 'should' => [], + 'must_not' => [ + [ + 'term' => ['_hidden' => true] + ], + [ + 'range' => [ + '_hiddenBeforeDateTime' => [ + 'gt' => 'now' + ] + ] + ], + [ + 'range' => [ + '_hiddenAfterDateTime' => [ + 'lt' => 'now' + ] + ] + ], + ], + ] + ] + ] + ], + 'fields' => ['__path'] + ]; + + /** + * {@inheritdoc} + */ + public function getCountRequestAsJSON() + { + $request = $this->request; + foreach ($this->unsupportedFieldsInCountRequest as $field) { + if (isset($request[$field])) { + unset($request[$field]); + } + } + + return json_encode($request); + } + + /** + * {@inheritdoc} + */ + public function size($size) + { + $this->request['size'] = (integer)$size; + } + + /** + * {@inheritdoc} + */ + public function from($size) + { + $this->request['from'] = (integer)$size; + } + + /** + * {@inheritdoc} + */ + public function fulltext($searchWord) + { + $this->appendAtPath('query.filtered.query.bool.must', [ + 'query_string' => [ + 'query' => $searchWord + ] + ]); + } + + /** + * {@inheritdoc} + */ + public function queryFilter($filterType, $filterOptions, $clauseType = 'must') + { + if (!in_array($clauseType, ['must', 'should', 'must_not'])) { + throw new Exception\QueryBuildingException('The given clause type "' . $clauseType . '" is not supported. Must be one of "must", "should", "must_not".', 1383716082); + } + + return $this->appendAtPath('query.filtered.filter.bool.' . $clauseType, [$filterType => $filterOptions]); + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php new file mode 100644 index 00000000..36f9c031 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php @@ -0,0 +1,24 @@ +request = $this->objectManager->get(QueryInterface::class, $this); - } - /** * HIGH-LEVEL API */ @@ -476,7 +469,7 @@ public function suggestions($name, array $suggestionDefinition) /** * Get the ElasticSearch request as we need it * - * @return \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface + * @return QueryInterface */ public function getRequest() { @@ -549,7 +542,6 @@ public function getFullElasticSearchHitForNode(NodeInterface $node) */ public function fetch() { - $request = $this->getRequest(); $timeBefore = microtime(true); $request = $this->request->getRequestAsJSON(); $response = $this->elasticSearchClient->getIndex()->request('GET', '/_search', [], $request); @@ -617,9 +609,8 @@ public function count() public function fulltext($searchWord) { // We automatically enable result highlighting when doing fulltext searches. It is up to the user to use this information or not use it. - $this->request - ->fulltext($searchWord) - ->highlight(150, 2); + $this->request->fulltext($searchWord); + $this->request->highlight(150, 2); return $this; } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php new file mode 100644 index 00000000..828c173c --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php @@ -0,0 +1,22 @@ +resolve('query'); + return new $className(); + } + + /** + * @return DocumentDriverInterface + */ + public function createDocumentDriver() + { + $className = $this->resolve('document'); + return new $className(); + } + + /** + * @return IndexerDriverInterface + */ + public function createIndexerDriver() + { + $className = $this->resolve('indexer'); + return new $className(); + } + + /** + * @return IndexerDriverInterface + */ + public function createIndexManagementDriver() + { + $className = $this->resolve('indexManagement'); + return new $className(); + } + + /** + * @return IndexerDriverInterface + */ + public function createRequestDriver() + { + $className = $this->resolve('request'); + return new $className(); + } + + /** + * @return IndexerDriverInterface + */ + public function createSystemDriver() + { + $className = $this->resolve('system'); + return new $className(); + } + + /** + * @param string $type + * @return mixed + * @throws DriverConfigurationException + */ + protected function resolve($type) + { + $version = trim($this->driverVersion); + if (trim($this->driverVersion) === '' || !isset($this->mapping[$version][$type])) { + throw new DriverConfigurationException(sprintf('Missing or wrongly configured driver type "%s" with the given version: %s', $type, $version ?: '[missing]'), 1485933538); + } + + $className = trim($this->mapping[$version][$type]); + + $this->logger->log(sprintf('Load %s implementation for Elastic %s (%s)', $type, $version, $className), LOG_DEBUG); + + return $className; + } + +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php index a5a52ff8..56a9aeef 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php @@ -11,6 +11,11 @@ * source code. */ +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexManagementDriverInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\RequestDriverInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\SystemDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder; @@ -65,6 +70,36 @@ class NodeIndexer extends AbstractNodeIndexer implements BulkNodeIndexerInterfac */ protected $contextFactory; + /** + * @var DocumentDriverInterface + * @Flow\Inject + */ + protected $documentDriver; + + /** + * @var IndexerDriverInterface + * @Flow\Inject + */ + protected $indexerDriver; + + /** + * @var IndexManagementDriverInterface + * @Flow\Inject + */ + protected $indexManagementDriver; + + /** + * @var RequestDriverInterface + * @Flow\Inject + */ + protected $requestDriver; + + /** + * @var SystemDriverInterface + * @Flow\Inject + */ + protected $systemDriver; + /** * The current ElasticSearch bulk request, in the format required by http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html * @@ -154,13 +189,12 @@ public function indexNode(NodeInterface $node, $targetWorkspaceName = null) if ($this->bulkProcessing === false) { // Remove document with the same contextPathHash but different NodeType, required after NodeType change $this->logger->log(sprintf('NodeIndexer (%s): Search and remove duplicate document for node %s (%s) if needed.', $documentIdentifier, $contextPath, $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); - $this->driver->deleteByDocumentIdentifier($this->getIndex(), $node, $documentIdentifier); + $this->documentDriver->deleteByDocumentIdentifier($this->getIndex(), $node, $documentIdentifier); } - $logger = $this->logger; $fulltextIndexOfNode = []; - $nodePropertiesToBeStoredInIndex = $this->extractPropertiesAndFulltext($node, $fulltextIndexOfNode, function ($propertyName) use ($logger, $documentIdentifier, $node) { - $logger->log(sprintf('NodeIndexer (%s) - Property "%s" not indexed because no configuration found, node type %s.', $documentIdentifier, $propertyName, $node->getNodeType()->getName()), LOG_DEBUG, null, 'ElasticSearch (CR)'); + $nodePropertiesToBeStoredInIndex = $this->extractPropertiesAndFulltext($node, $fulltextIndexOfNode, function ($propertyName) use ($documentIdentifier, $node) { + $this->logger->log(sprintf('NodeIndexer (%s) - Property "%s" not indexed because no configuration found, node type %s.', $documentIdentifier, $propertyName, $node->getNodeType()->getName()), LOG_DEBUG, null, 'ElasticSearch (CR)'); }); $document = new ElasticSearchDocument($mappingType, @@ -180,11 +214,8 @@ public function indexNode(NodeInterface $node, $targetWorkspaceName = null) } if ($this->isFulltextEnabled($node)) { - $query = $this->driver->fulltext($node, $fulltextIndexOfNode, $targetWorkspaceName); - if ($query !== null) { - // todo driver::fulltext must always return an array, extract decision logic to a dedicated method - $this->currentBulkRequest[] = $query; - } + $this->currentBulkRequest[] = $this->indexerDriver->document($this->getIndexName(), $node, $document, $documentData, $fulltextIndexOfNode, $targetWorkspaceName); + $this->currentBulkRequest[] = $this->indexerDriver->fulltext($node, $fulltextIndexOfNode, $targetWorkspaceName); } $this->logger->log(sprintf('NodeIndexer (%s): Indexed node %s.', $documentIdentifier, $contextPath), LOG_DEBUG, null, 'ElasticSearch (CR)'); @@ -260,8 +291,8 @@ public function removeNode(NodeInterface $node, $targetWorkspaceName = null) $documentIdentifier = $this->calculateDocumentIdentifier($node, $targetWorkspaceName); - $this->currentBulkRequest[] = $this->driver->delete($node, $documentIdentifier); - $this->currentBulkRequest[] = $this->driver->fulltext($node, [], $targetWorkspaceName); + $this->currentBulkRequest[] = $this->documentDriver->delete($node, $documentIdentifier); + $this->currentBulkRequest[] = $this->indexerDriver->fulltext($node, [], $targetWorkspaceName); $this->logger->log(sprintf('NodeIndexer (%s): Removed node %s (%s) from index.', $documentIdentifier, $node->getContextPath(), $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); } @@ -273,12 +304,13 @@ public function removeNode(NodeInterface $node, $targetWorkspaceName = null) */ public function flush() { - if (count($this->currentBulkRequest) === 0) { + $bulkRequest = array_filter($this->currentBulkRequest); + if (count($bulkRequest) === 0) { return; } $content = ''; - foreach ($this->currentBulkRequest as $bulkRequestTuple) { + foreach ($bulkRequest as $bulkRequestTuple) { $tupleAsJson = ''; foreach ($bulkRequestTuple as $bulkRequestItem) { $itemAsJson = json_encode($bulkRequestItem); @@ -292,10 +324,10 @@ public function flush() } if ($content !== '') { - $response = $this->driver->bulk($this->getIndex(), $content); + $response = $this->requestDriver->bulk($this->getIndex(), $content); foreach ($response as $responseLine) { - if (!is_object($response) || (isset($response->errors) && $response->errors !== false)) { - $this->logger->log('NodeIndexer: ' . $responseLine, LOG_ERR, null, 'ElasticSearch (CR)'); + if (isset($response['errors']) && $response['errors'] !== false) { + $this->logger->log('NodeIndexer: ' . json_encode($responseLine), LOG_ERR, null, 'ElasticSearch (CR)'); } } } @@ -324,11 +356,11 @@ public function updateIndexAlias() $aliasActions = []; try { - $indexNames = $this->driver->indexNames($aliasName); + $indexNames = $this->indexManagementDriver->indexesByAlias($aliasName); if ($indexNames === []) { // if there is an actual index with the name we want to use as alias, remove it now - $this->driver->removeAlias($aliasName); + $this->indexManagementDriver->remove($aliasName); } else { foreach ($indexNames as $indexName) { $aliasActions[] = [ @@ -353,7 +385,7 @@ public function updateIndexAlias() ] ]; - $this->driver->aliasActions($aliasActions); + $this->indexManagementDriver->actions($aliasActions); } /** @@ -366,9 +398,9 @@ public function removeOldIndices() { $aliasName = $this->searchClient->getIndexName(); // The alias name is the unprefixed index name - $currentlyLiveIndices = $this->driver->currentlyLiveIndices($aliasName); + $currentlyLiveIndices = $this->indexManagementDriver->indexesByAlias($aliasName); - $indexStatus = $this->driver->status(); + $indexStatus = $this->systemDriver->status(); $allIndices = array_keys($indexStatus['indices']); $indicesToBeRemoved = []; @@ -387,7 +419,7 @@ public function removeOldIndices() $indicesToBeRemoved[] = $indexName; } - $this->driver->deleteIndices($indicesToBeRemoved); + $this->indexManagementDriver->delete($indicesToBeRemoved); return $indicesToBeRemoved; } diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml index 56b79f21..01df676c 100644 --- a/Configuration/Objects.yaml +++ b/Configuration/Objects.yaml @@ -1,5 +1,32 @@ Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface: - className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Query\FilteredQuery' + scope: prototype + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' + factoryMethodName: createQuery + +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface: + scope: singleton + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' + factoryMethodName: createDocumentDriver + +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface: + scope: singleton + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' + factoryMethodName: createIndexerDriver + +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexManagementDriverInterface: + scope: singleton + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' + factoryMethodName: createIndexManagementDriver + +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\RequestDriverInterface: + scope: singleton + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' + factoryMethodName: createRequestDriver + +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\SystemDriverInterface: + scope: singleton + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' + factoryMethodName: createSystemDriver Flowpack\ElasticSearch\ContentRepositoryAdaptor\Command\NodeIndexCommandController: properties: @@ -36,5 +63,5 @@ Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface: Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient: scope: singleton - factoryObjectName: Flowpack\ElasticSearch\ContentRepositoryAdaptor\Client\ClientFactory + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Client\ClientFactory' factoryMethodName: create diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 79938420..2009420b 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -1,3 +1,24 @@ +Flowpack: + ElasticSearch: + ContentRepositoryAdaptor: + driver: + version: '2.x' + mapping: + '1.x': + query: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Query\FilteredQuery' + document: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\DocumentDriver' + indexer: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexerDriver' + indexManagement: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexManagementDriver' + request: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\RequestDriver' + system: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\SystemDriver' + '2.x': + query: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\Query\FilteredQuery' + document: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\DocumentDriver' + indexer: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexerDriver' + indexManagement: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexManagementDriver' + request: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\RequestDriver' + system: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\SystemDriver' + TYPO3: TYPO3CR: Search: @@ -65,4 +86,4 @@ TYPO3: 'reference': elasticSearchMapping: type: string - index: not_analyzed \ No newline at end of file + index: not_analyzed From 6b7d15b2deb2290386fdc1678e492e98c1ae7a6e Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 22:29:11 +0100 Subject: [PATCH 08/41] TASK: Extend the FilteredQuery from Driver Version1 in Version2 --- .../Driver/Version2/Query/FilteredQuery.php | 108 +----------------- 1 file changed, 3 insertions(+), 105 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php index 7bb9a51b..507a9f2d 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php @@ -11,111 +11,9 @@ * source code. */ -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractQuery; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1; -class FilteredQuery extends AbstractQuery +class FilteredQuery extends Version1\Query\FilteredQuery { - /** - * The ElasticSearch request, as it is being built up. - * - * @var array - */ - protected $request = [ - 'query' => [ - 'filtered' => [ - 'query' => [ - 'bool' => [ - 'must' => [ - [ - 'match_all' => [] - ] - ] - ] - - ], - 'filter' => [ - 'bool' => [ - 'must' => [], - 'should' => [], - 'must_not' => [ - [ - 'term' => ['_hidden' => true] - ], - [ - 'range' => [ - '_hiddenBeforeDateTime' => [ - 'gt' => 'now' - ] - ] - ], - [ - 'range' => [ - '_hiddenAfterDateTime' => [ - 'lt' => 'now' - ] - ] - ], - ], - ] - ] - ] - ], - 'fields' => ['__path'] - ]; - - /** - * {@inheritdoc} - */ - public function getCountRequestAsJSON() - { - $request = $this->request; - foreach ($this->unsupportedFieldsInCountRequest as $field) { - if (isset($request[$field])) { - unset($request[$field]); - } - } - - return json_encode($request); - } - - /** - * {@inheritdoc} - */ - public function size($size) - { - $this->request['size'] = (integer)$size; - } - - /** - * {@inheritdoc} - */ - public function from($size) - { - $this->request['from'] = (integer)$size; - } - - /** - * {@inheritdoc} - */ - public function fulltext($searchWord) - { - $this->appendAtPath('query.filtered.query.bool.must', [ - 'query_string' => [ - 'query' => $searchWord - ] - ]); - } - - /** - * {@inheritdoc} - */ - public function queryFilter($filterType, $filterOptions, $clauseType = 'must') - { - if (!in_array($clauseType, ['must', 'should', 'must_not'])) { - throw new Exception\QueryBuildingException('The given clause type "' . $clauseType . '" is not supported. Must be one of "must", "should", "must_not".', 1383716082); - } - - return $this->appendAtPath('query.filtered.filter.bool.' . $clauseType, [$filterType => $filterOptions]); - } + } From 8fe6a00f342f3f3b3526bb8db805ea3bdf8a9c0b Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 22:40:02 +0100 Subject: [PATCH 09/41] TASK: Add AbstractIndexerDriver --- .../Driver/AbstractDriver.php | 18 -------- .../Driver/AbstractIndexerDriver.php | 41 +++++++++++++++++++ .../Driver/Version1/IndexerDriver.php | 5 ++- 3 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php index 51bfe34e..4bd6d8fc 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php @@ -32,22 +32,4 @@ abstract class AbstractDriver * @var LoggerInterface */ protected $logger; - - /** - * Whether the node is configured as fulltext root. - * - * @param NodeInterface $node - * @return boolean - */ - protected function isFulltextRoot(NodeInterface $node) - { - if ($node->getNodeType()->hasConfiguration('search')) { - $elasticSearchSettingsForNode = $node->getNodeType()->getConfiguration('search'); - if (isset($elasticSearchSettingsForNode['fulltext']['isRoot']) && $elasticSearchSettingsForNode['fulltext']['isRoot'] === true) { - return true; - } - } - - return false; - } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php new file mode 100644 index 00000000..e0ce3d68 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php @@ -0,0 +1,41 @@ +getNodeType()->hasConfiguration('search')) { + $elasticSearchSettingsForNode = $node->getNodeType()->getConfiguration('search'); + if (isset($elasticSearchSettingsForNode['fulltext']['isRoot']) && $elasticSearchSettingsForNode['fulltext']['isRoot'] === true) { + return true; + } + } + + return false; + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php index c15322b4..060a550b 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php @@ -12,6 +12,7 @@ */ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractDriver; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractIndexerDriver; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder; @@ -20,11 +21,11 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Fulltext Indexer Driver for Elastic version 1.x + * Indexer Driver for Elastic version 1.x * * @Flow\Scope("singleton") */ -class IndexerDriver extends AbstractDriver implements IndexerDriverInterface +class IndexerDriver extends AbstractIndexerDriver implements IndexerDriverInterface { /** * {@inheritdoc} From 123e09998e2951df943642aaa77680d77c723877 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 22:40:22 +0100 Subject: [PATCH 10/41] TASK: Better PHP doc in Elastic Drivers --- .../Driver/Version1/DocumentDriver.php | 2 +- .../Driver/Version1/IndexManagementDriver.php | 2 +- .../Driver/Version1/Query/FilteredQuery.php | 3 +++ .../Driver/Version1/RequestDriver.php | 2 +- .../Driver/Version1/SystemDriver.php | 2 +- .../Driver/Version2/DocumentDriver.php | 2 +- .../Driver/Version2/IndexManagementDriver.php | 2 +- .../Driver/Version2/IndexerDriver.php | 2 +- .../Driver/Version2/Query/FilteredQuery.php | 5 ++++- .../Driver/Version2/RequestDriver.php | 2 +- .../Driver/Version2/SystemDriver.php | 2 +- 11 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php index 817f8a39..5b73d678 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php @@ -20,7 +20,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Fulltext Indexer Driver for Elastic version 1.x + * Document Driver for Elastic version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php index 2add9fc2..ba69e609 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php @@ -18,7 +18,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Fulltext Indexer Driver for Elastic version 1.x + * Index Management Driver for Elastic version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php index 369afa99..92498097 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php @@ -14,6 +14,9 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractQuery; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; +/** + * Default Filtered Query + */ class FilteredQuery extends AbstractQuery { /** diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php index 0155947d..ba759afa 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php @@ -17,7 +17,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Fulltext Indexer Driver for Elastic version 1.x + * Request Driver for Elastic version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php index 05f1e959..8d27fd90 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php @@ -16,7 +16,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Fulltext Indexer Driver for Elastic version 1.x + * System Driver for Elastic version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php index 2ad66d5e..d1c151c5 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php @@ -18,7 +18,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Fulltext Indexer Driver for Elastic version 2.x + * Document Driver for Elastic version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php index ae783e8b..f5ce1225 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php @@ -15,7 +15,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Fulltext Indexer Driver for Elastic version 2.x + * Index Management Driver for Elastic version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php index a3ca2333..0505af0c 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php @@ -18,7 +18,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Fulltext Indexer Driver for Elastic version 2.x + * Indexer Driver for Elastic version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php index 507a9f2d..b5faca46 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php @@ -13,7 +13,10 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1; +/** + * Default Filtered Query + */ class FilteredQuery extends Version1\Query\FilteredQuery { - + } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php index 36f9c031..c4736e64 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php @@ -15,7 +15,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Fulltext Indexer Driver for Elastic version 2.x + * Request Driver for Elastic version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php index b8a27c4e..daa9948e 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php @@ -15,7 +15,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Fulltext Indexer Driver for Elastic version 2.x + * System Driver for Elastic version 2.x * * @Flow\Scope("singleton") */ From c536a676d6564ff9f29db1da7bdeea61623b03ca Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 22:52:51 +0100 Subject: [PATCH 11/41] TASK: Rename IndexManagementDriverInterface:actions to aliasActions --- .../Driver/IndexManagementDriverInterface.php | 2 +- .../Driver/Version1/IndexManagementDriver.php | 2 +- .../ContentRepositoryAdaptor/Indexer/NodeIndexer.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php index 287250ae..9d408cc4 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php @@ -46,5 +46,5 @@ public function remove($aliasName); * @param array $actions * @return mixed */ - public function actions(array $actions); + public function aliasActions(array $actions); } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php index ba69e609..53920b49 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php @@ -38,7 +38,7 @@ public function delete(array $indices) /** * {@inheritdoc} */ - public function actions(array $actions) + public function aliasActions(array $actions) { $this->searchClient->request('POST', '/_aliases', [], \json_encode(['actions' => $actions])); } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php index 56a9aeef..3bb13e4c 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php @@ -385,7 +385,7 @@ public function updateIndexAlias() ] ]; - $this->indexManagementDriver->actions($aliasActions); + $this->indexManagementDriver->aliasActions($aliasActions); } /** From 95192f4740997205e9d6a701ff6dacee01e7c839 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 22:54:05 +0100 Subject: [PATCH 12/41] TASK: Rename IndexManagementDriverInterface:remove to deleteIndex --- .../Driver/IndexManagementDriverInterface.php | 2 +- .../Driver/Version1/IndexManagementDriver.php | 2 +- .../ContentRepositoryAdaptor/Indexer/NodeIndexer.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php index 9d408cc4..5978956a 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php @@ -38,7 +38,7 @@ public function indexesByAlias($alias); * @param string $aliasName * @return void */ - public function remove($aliasName); + public function deleteIndex($aliasName); /** * Execute batch aliases actions diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php index 53920b49..6a53636c 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php @@ -46,7 +46,7 @@ public function aliasActions(array $actions) /** * {@inheritdoc} */ - public function remove($aliasName) + public function deleteIndex($aliasName) { $response = $this->searchClient->request('HEAD', '/' . $aliasName); if ($response->getStatusCode() === 200) { diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php index 3bb13e4c..98163ea1 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php @@ -360,7 +360,7 @@ public function updateIndexAlias() if ($indexNames === []) { // if there is an actual index with the name we want to use as alias, remove it now - $this->indexManagementDriver->remove($aliasName); + $this->indexManagementDriver->deleteIndex($aliasName); } else { foreach ($indexNames as $indexName) { $aliasActions[] = [ From 8cf86eeea4cbc63ffc70be255e6588ad3f116da9 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 23:00:30 +0100 Subject: [PATCH 13/41] TASK: Add a section in the README about the Driver settings --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 18c2b017..9dcab74b 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,21 @@ # Neos Elasticsearch Adapter -*supporting Elasticsearch versions 1.2.x to 1.7.x* - -Created by Sebastian Kurfürst; [contributions by Karsten Dambekalns, Robert Lemke and others](https://github.com/Flowpack/Flowpack.ElasticSearch.ContentRepositoryAdaptor/graphs/contributors). - This project connects the Neos Content Repository (TYPO3CR) to Elasticsearch; enabling two main functionalities: * finding Nodes in TypoScript / Eel by arbitrary queries * Full-Text Indexing of Pages and other Documents (of course including the full content) +## Elastic version support + +You can switch the Elastic driver by editing ```Settings.yaml``` +(```Flowpack.ElasticSearchContentRepositoryAdaptor.driver.version```) with the following value: + +* ```1.x``` to support Elastic 1.2 to 1.7 +* ```2.x``` to support Elastic 2.x to 1.4 + +_Currently the Driver interfaces as not marked as API, and can be changed to adapt to future needs (especially the support of Elastic v5)._ ## Relevant Packages @@ -23,7 +28,6 @@ main functionalities: instead of this package); storing the search index in SQLite * [Flowpack.SearchPlugin](https://www.neos.io/download-and-extend/packages/flowpack/flowpack-searchplugin.html): search plugin for Neos - ## Installation ``` @@ -713,3 +717,4 @@ The configuration from Version 1 to Version 2 has changed; here's what to change 3. Replace `ElasticSeach.fulltext` by `Indexing` 4. Search for `ElasticSearch.` (inside the `indexing` expressions) and replace them by `Indexing.` +Created by Sebastian Kurfürst; [contributions by Karsten Dambekalns, Robert Lemke and others](https://github.com/Flowpack/Flowpack.ElasticSearch.ContentRepositoryAdaptor/graphs/contributors). From 09fc49b728fed889199dfb9635ad2bdeed06531c Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 23:01:51 +0100 Subject: [PATCH 14/41] TASK: Fix styling issues --- .../Driver/Version2/Query/FilteredQuery.php | 1 - .../ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php | 1 - .../ContentRepositoryAdaptor/Factory/DriverFactory.php | 1 - 3 files changed, 3 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php index b5faca46..fd14cdca 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FilteredQuery.php @@ -18,5 +18,4 @@ */ class FilteredQuery extends Version1\Query\FilteredQuery { - } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php index daa9948e..c5a6bf12 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php @@ -21,5 +21,4 @@ */ class SystemDriver extends Version1\SystemDriver { - } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php index dd17f2d6..25cf0ae2 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php @@ -116,5 +116,4 @@ protected function resolve($type) return $className; } - } From c51fcfdde51e32f688a5d0318e1ca05c9d2d7a96 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Wed, 1 Feb 2017 23:10:42 +0100 Subject: [PATCH 15/41] TASK: Rename DocumentDriverInterface::deleteByDocumentIdentifier --- .../Driver/DocumentDriverInterface.php | 5 +++-- .../Driver/Version1/DocumentDriver.php | 5 +++-- .../Driver/Version2/DocumentDriver.php | 10 ++++------ .../ContentRepositoryAdaptor/Indexer/NodeIndexer.php | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php index 15b8cc0b..fbfab6b5 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php @@ -12,6 +12,7 @@ */ use Flowpack\ElasticSearch\Domain\Model\Index; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; +use TYPO3\TYPO3CR\Domain\Model\NodeType; /** * Document Driver Interface @@ -31,9 +32,9 @@ public function delete(NodeInterface $node, $identifier); * Generate the query to delete Elastic Document by Document Identifier but skip Document with the same Node Type * * @param Index $index - * @param NodeInterface $node * @param string $documentIdentifier + * @param NodeType $nodeType * @return array */ - public function deleteByDocumentIdentifier(Index $index, NodeInterface $node, $documentIdentifier); + public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentIdentifier, NodeType $nodeType); } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php index 5b73d678..5864cc67 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php @@ -18,6 +18,7 @@ use Flowpack\ElasticSearch\Domain\Model\Index; use TYPO3\Flow\Annotations as Flow; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; +use TYPO3\TYPO3CR\Domain\Model\NodeType; /** * Document Driver for Elastic version 1.x @@ -44,7 +45,7 @@ public function delete(NodeInterface $node, $identifier) /** * {@inheritdoc} */ - public function deleteByDocumentIdentifier(Index $index, NodeInterface $node, $documentIdentifier) + public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentIdentifier, NodeType $nodeType) { $index->request('DELETE', '/_query', [], json_encode([ 'query' => [ @@ -56,7 +57,7 @@ public function deleteByDocumentIdentifier(Index $index, NodeInterface $node, $d ], 'must_not' => [ 'term' => [ - '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName()) + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($nodeType->getName()) ] ], ] diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php index d1c151c5..1a42eed6 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php @@ -16,6 +16,7 @@ use Flowpack\ElasticSearch\Domain\Model\Index; use TYPO3\Flow\Annotations as Flow; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; +use TYPO3\TYPO3CR\Domain\Model\NodeType; /** * Document Driver for Elastic version 2.x @@ -25,13 +26,10 @@ class DocumentDriver extends Version1\DocumentDriver { /** - * @param Index $index - * @param NodeInterface $node - * @param string $documentIdentifier + * {@inheritdoc} */ - public function deleteByDocumentIdentifier(Index $index, NodeInterface $node, $documentIdentifier) + public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentIdentifier, NodeType $nodeType) { - $type = NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName()); $result = $index->request('GET', '/_search?scroll=1m', [], json_encode([ 'sort' => ['_doc'], 'query' => [ @@ -43,7 +41,7 @@ public function deleteByDocumentIdentifier(Index $index, NodeInterface $node, $d ], 'must_not' => [ 'term' => [ - '_type' => $type + '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($nodeType->getName()) ] ] ] diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php index 98163ea1..fad0273a 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php @@ -189,7 +189,7 @@ public function indexNode(NodeInterface $node, $targetWorkspaceName = null) if ($this->bulkProcessing === false) { // Remove document with the same contextPathHash but different NodeType, required after NodeType change $this->logger->log(sprintf('NodeIndexer (%s): Search and remove duplicate document for node %s (%s) if needed.', $documentIdentifier, $contextPath, $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); - $this->documentDriver->deleteByDocumentIdentifier($this->getIndex(), $node, $documentIdentifier); + $this->documentDriver->deleteDuplicateDocumentNotMatchingType($this->getIndex(), $documentIdentifier, $node->getNodeType()); } $fulltextIndexOfNode = []; From 36fb60fa0b0e9a1be6029b5c40119d3f5073d55c Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 3 Feb 2017 15:49:26 +0100 Subject: [PATCH 16/41] BUGIFX: Bulk request must end with end of line caracter --- .../Driver/Version1/RequestDriver.php | 3 +++ .../Driver/Version2/DocumentDriver.php | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php index ba759afa..f95a0b05 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php @@ -32,6 +32,9 @@ public function bulk(Index $index, $request) $request = json_encode($request); } + // Bulk request MUST end with line return + $request = trim($request) . "\n"; + $response = $index->request('POST', '/_bulk', [], $request)->getOriginalResponse()->getContent(); return array_map(function ($line) { diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php index 1a42eed6..55d2255b 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php @@ -50,7 +50,7 @@ public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentId $treatedContent = $result->getTreatedContent(); $scrollId = $treatedContent['_scroll_id']; $mapHitToDeleteRequest = function ($hit) { - $bulkRequest[] = json_encode([ + return json_encode([ 'delete' => [ '_type' => $hit['_type'], '_id' => $hit['_id'] @@ -66,7 +66,7 @@ public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentId } $this->logger->log(sprintf('NodeIndexer: Check duplicate nodes for %s (%s), found %d document(s)', $documentIdentifier, $type, count($bulkRequest)), LOG_DEBUG, null, 'ElasticSearch (CR)'); if ($bulkRequest !== []) { - $index->request('POST', '/_bulk', [], implode("\n", $bulkRequest)); + $index->request('POST', '/_bulk', [], implode("\n", $bulkRequest) . "\n"); } $this->searchClient->request('DELETE', '/_search/scroll', [], json_encode([ 'scroll_id' => [ From 3056c07d04a18abe341d5e90b57234f18419571d Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 3 Feb 2017 16:00:46 +0100 Subject: [PATCH 17/41] TASK: Update documentation about custom index name --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 9dcab74b..24b52762 100644 --- a/README.md +++ b/README.md @@ -678,6 +678,19 @@ in the NodeTypes.yaml. Generally this works by defining the global mapping at `[ search_analyzer: custom_french_analyzer ``` +## Change the default Elastic index name + +If you need to run serveral (different) neos instances on the same elasticsearch server you will need to change the Configuration/Settings.yaml indexName for each of your project. + +So `./flow nodeindex:build` or `./flow nodeindex:cleanup` won't overwrite your other sites index. + +``` +TYPO3: + TYPO3CR: + Search: + elasticSearch: + indexName: useMoreSpecificIndexName +``` ## Debugging From 1e1523c36948911368a8b7ce7e8db8be8dbf511b Mon Sep 17 00:00:00 2001 From: Niklas Droste Date: Sat, 4 Feb 2017 11:06:29 +0100 Subject: [PATCH 18/41] [BUGFIX] Change wrong variable name --- .../ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php index 55d2255b..14c690dc 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php @@ -64,7 +64,7 @@ public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentId $result = $index->request('GET', '/_search/scroll?scroll=1m', [], $scrollId, false); $treatedContent = $result->getTreatedContent(); } - $this->logger->log(sprintf('NodeIndexer: Check duplicate nodes for %s (%s), found %d document(s)', $documentIdentifier, $type, count($bulkRequest)), LOG_DEBUG, null, 'ElasticSearch (CR)'); + $this->logger->log(sprintf('NodeIndexer: Check duplicate nodes for %s (%s), found %d document(s)', $documentIdentifier, $nodeType->getName(), count($bulkRequest)), LOG_DEBUG, null, 'ElasticSearch (CR)'); if ($bulkRequest !== []) { $index->request('POST', '/_bulk', [], implode("\n", $bulkRequest) . "\n"); } From 88bbb314858d21ce566a81029516b742543798bd Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Sat, 4 Feb 2017 11:09:55 +0100 Subject: [PATCH 19/41] TASK: Fix Unit tests by not expecting should/must_not term --- Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php index 6e46c4c8..5f72e348 100644 --- a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php +++ b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php @@ -59,9 +59,7 @@ public function basicRequestStructureTakesContextNodeIntoAccount() 'bool' => [ 'must' => [ ['match_all' => []] - ], - 'should' => [], - 'must_not' => [] + ] ] ], 'filter' => [ @@ -113,7 +111,7 @@ public function basicRequestStructureTakesContextNodeIntoAccount() 'fields' => ['__path'] ]; $actual = $this->queryBuilder->getRequest()->toArray(); - $this->assertSame($expected, $actual); + $this->assertEquals($expected, $actual); } /** From 71eb4aeda9bd381b2b00fb9142bfe44f36dce9f3 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Sat, 4 Feb 2017 13:01:29 +0100 Subject: [PATCH 20/41] TASK: Extract query request and configuration to Settings With this change it's more easy to add small change in the query by just overriding the settings. --- .../Driver/AbstractQuery.php | 12 ++- .../Driver/Version1/Query/FilteredQuery.php | 48 ------------ .../Factory/DriverFactory.php | 27 +++---- Configuration/Settings.yaml | 74 ++++++++++++++++--- 4 files changed, 85 insertions(+), 76 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php index 15959a31..98283ff2 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php @@ -32,7 +32,17 @@ abstract class AbstractQuery implements QueryInterface, \JsonSerializable, \Arra * * @var array */ - protected $unsupportedFieldsInCountRequest = ['fields', 'sort', 'from', 'size', 'highlight', 'aggs', 'aggregations']; + protected $unsupportedFieldsInCountRequest = []; + + /** + * @param array $request + * @param array $unsupportedFieldsInCountRequest + */ + public function __construct(array $request, array $unsupportedFieldsInCountRequest) + { + $this->request = $request; + $this->unsupportedFieldsInCountRequest = $unsupportedFieldsInCountRequest; + } /** * Modify a part of the Elasticsearch Request denoted by $path, merging together diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php index 92498097..f62f5ba8 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php @@ -19,54 +19,6 @@ */ class FilteredQuery extends AbstractQuery { - /** - * The ElasticSearch request, as it is being built up. - * - * @var array - */ - protected $request = [ - 'query' => [ - 'filtered' => [ - 'query' => [ - 'bool' => [ - 'must' => [ - [ - 'match_all' => [] - ] - ] - ] - - ], - 'filter' => [ - 'bool' => [ - 'must' => [], - 'should' => [], - 'must_not' => [ - [ - 'term' => ['_hidden' => true] - ], - [ - 'range' => [ - '_hiddenBeforeDateTime' => [ - 'gt' => 'now' - ] - ] - ], - [ - 'range' => [ - '_hiddenAfterDateTime' => [ - 'lt' => 'now' - ] - ] - ], - ], - ] - ] - ] - ], - 'fields' => ['__path'] - ]; - /** * {@inheritdoc} */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php index 25cf0ae2..d1651167 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php @@ -49,8 +49,7 @@ class DriverFactory */ public function createQuery() { - $className = $this->resolve('query'); - return new $className(); + return $this->resolve('query'); } /** @@ -58,8 +57,7 @@ public function createQuery() */ public function createDocumentDriver() { - $className = $this->resolve('document'); - return new $className(); + return $this->resolve('document'); } /** @@ -67,8 +65,7 @@ public function createDocumentDriver() */ public function createIndexerDriver() { - $className = $this->resolve('indexer'); - return new $className(); + return $this->resolve('indexer'); } /** @@ -76,8 +73,7 @@ public function createIndexerDriver() */ public function createIndexManagementDriver() { - $className = $this->resolve('indexManagement'); - return new $className(); + return $this->resolve('indexManagement'); } /** @@ -85,8 +81,7 @@ public function createIndexManagementDriver() */ public function createRequestDriver() { - $className = $this->resolve('request'); - return new $className(); + return $this->resolve('request'); } /** @@ -94,8 +89,7 @@ public function createRequestDriver() */ public function createSystemDriver() { - $className = $this->resolve('system'); - return new $className(); + return $this->resolve('system'); } /** @@ -106,14 +100,17 @@ public function createSystemDriver() protected function resolve($type) { $version = trim($this->driverVersion); - if (trim($this->driverVersion) === '' || !isset($this->mapping[$version][$type])) { + if (trim($this->driverVersion) === '' || !isset($this->mapping[$version][$type]['className'])) { throw new DriverConfigurationException(sprintf('Missing or wrongly configured driver type "%s" with the given version: %s', $type, $version ?: '[missing]'), 1485933538); } - $className = trim($this->mapping[$version][$type]); + $className = trim($this->mapping[$version][$type]['className']); $this->logger->log(sprintf('Load %s implementation for Elastic %s (%s)', $type, $version, $className), LOG_DEBUG); - return $className; + if (!isset($this->mapping[$version][$type]['arguments'])) { + return new $className(); + } + return new $className(...array_values($this->mapping[$version][$type]['arguments'])); } } diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 2009420b..47adb212 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -5,19 +5,69 @@ Flowpack: version: '2.x' mapping: '1.x': - query: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Query\FilteredQuery' - document: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\DocumentDriver' - indexer: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexerDriver' - indexManagement: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexManagementDriver' - request: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\RequestDriver' - system: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\SystemDriver' + query: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Query\FilteredQuery' + arguments: + request: &request_version1 + query: + filtered: + query: + bool: + must: + - match_all: [] + filter: + bool: + must: [] + should: [] + must_not: + - term: + _hidden: true + - range: + _hiddenBeforeDateTime: + gt: now + - range: + _hiddenAfterDateTime: + lt: now + fields: + - '__path' + unsupportedFieldsInCountRequest: &unsupportedFieldsInCountRequest_version1 + - 'fields' + - 'sort' + - 'from' + - 'size' + - 'highlight' + - 'aggs' + - 'aggregations' + + document: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\DocumentDriver' + indexer: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexerDriver' + indexManagement: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexManagementDriver' + request: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\RequestDriver' + system: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\SystemDriver' + '2.x': - query: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\Query\FilteredQuery' - document: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\DocumentDriver' - indexer: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexerDriver' - indexManagement: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexManagementDriver' - request: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\RequestDriver' - system: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\SystemDriver' + query: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\Query\FilteredQuery' + arguments: + request: + <<: *request_version1 + unsupportedFieldsInCountRequest: + <<: *unsupportedFieldsInCountRequest_version1 + document: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\DocumentDriver' + indexer: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexerDriver' + indexManagement: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexManagementDriver' + request: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\RequestDriver' + system: + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\SystemDriver' TYPO3: TYPO3CR: From 61e376e9559776642edc77cd1512d1670aecb38e Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Sat, 4 Feb 2017 13:05:58 +0100 Subject: [PATCH 21/41] TASK: Refactor IndexDriver interface --- ...Interface.php => IndexDriverInterface.php} | 16 ++++--------- ...exManagementDriver.php => IndexDriver.php} | 23 +++++-------------- ...exManagementDriver.php => IndexDriver.php} | 2 +- .../Indexer/NodeIndexer.php | 18 ++++++++------- Configuration/Objects.yaml | 2 +- Configuration/Settings.yaml | 4 ++-- 6 files changed, 24 insertions(+), 41 deletions(-) rename Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/{IndexManagementDriverInterface.php => IndexDriverInterface.php} (72%) rename Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/{IndexManagementDriver.php => IndexDriver.php} (74%) rename Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/{IndexManagementDriver.php => IndexDriver.php} (90%) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexDriverInterface.php similarity index 72% rename from Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php rename to Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexDriverInterface.php index 5978956a..a19fbaab 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexManagementDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexDriverInterface.php @@ -12,18 +12,10 @@ */ /** - * Elastic Index Management Driver Interface + * Elastic Index Driver Interface */ -interface IndexManagementDriverInterface +interface IndexDriverInterface { - /** - * Delete given indexes - * - * @param array $indices - * @return void - */ - public function delete(array $indices); - /** * Get the list of Indexes attached to the given alias * @@ -35,10 +27,10 @@ public function indexesByAlias($alias); /** * Remove alias by name * - * @param string $aliasName + * @param string $index * @return void */ - public function deleteIndex($aliasName); + public function deleteIndex($index); /** * Execute batch aliases actions diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexDriver.php similarity index 74% rename from Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php rename to Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexDriver.php index 6a53636c..b7b64d87 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexManagementDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexDriver.php @@ -13,7 +13,7 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractDriver; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexManagementDriverInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; use TYPO3\Flow\Annotations as Flow; @@ -22,19 +22,8 @@ * * @Flow\Scope("singleton") */ -class IndexManagementDriver extends AbstractDriver implements IndexManagementDriverInterface +class IndexDriver extends AbstractDriver implements IndexDriverInterface { - /** - * {@inheritdoc} - */ - public function delete(array $indices) - { - if (count($indices) === 0) { - return; - } - $this->searchClient->request('DELETE', '/' . implode(',', $indices) . '/'); - } - /** * {@inheritdoc} */ @@ -46,13 +35,13 @@ public function aliasActions(array $actions) /** * {@inheritdoc} */ - public function deleteIndex($aliasName) + public function deleteIndex($index) { - $response = $this->searchClient->request('HEAD', '/' . $aliasName); + $response = $this->searchClient->request('HEAD', '/' . $index); if ($response->getStatusCode() === 200) { - $response = $this->searchClient->request('DELETE', '/' . $aliasName); + $response = $this->searchClient->request('DELETE', '/' . $index); if ($response->getStatusCode() !== 200) { - throw new Exception('The index "' . $aliasName . '" could not be removed to be replaced by an alias. (return code: ' . $response->getStatusCode() . ')', 1395419177); + throw new Exception('The index "' . $index . '" could not be deleted. (return code: ' . $response->getStatusCode() . ')', 1395419177); } } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexDriver.php similarity index 90% rename from Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php rename to Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexDriver.php index f5ce1225..8394ed9a 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexManagementDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexDriver.php @@ -19,6 +19,6 @@ * * @Flow\Scope("singleton") */ -class IndexManagementDriver extends Version1\IndexManagementDriver +class IndexDriver extends Version1\IndexDriver { } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php index fad0273a..d27943ce 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php @@ -13,7 +13,7 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexManagementDriverInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\RequestDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\SystemDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; @@ -83,10 +83,10 @@ class NodeIndexer extends AbstractNodeIndexer implements BulkNodeIndexerInterfac protected $indexerDriver; /** - * @var IndexManagementDriverInterface + * @var IndexDriverInterface * @Flow\Inject */ - protected $indexManagementDriver; + protected $indexDriver; /** * @var RequestDriverInterface @@ -356,11 +356,11 @@ public function updateIndexAlias() $aliasActions = []; try { - $indexNames = $this->indexManagementDriver->indexesByAlias($aliasName); + $indexNames = $this->indexDriver->indexesByAlias($aliasName); if ($indexNames === []) { // if there is an actual index with the name we want to use as alias, remove it now - $this->indexManagementDriver->deleteIndex($aliasName); + $this->indexDriver->deleteIndex($aliasName); } else { foreach ($indexNames as $indexName) { $aliasActions[] = [ @@ -385,7 +385,7 @@ public function updateIndexAlias() ] ]; - $this->indexManagementDriver->aliasActions($aliasActions); + $this->indexDriver->aliasActions($aliasActions); } /** @@ -398,7 +398,7 @@ public function removeOldIndices() { $aliasName = $this->searchClient->getIndexName(); // The alias name is the unprefixed index name - $currentlyLiveIndices = $this->indexManagementDriver->indexesByAlias($aliasName); + $currentlyLiveIndices = $this->indexDriver->indexesByAlias($aliasName); $indexStatus = $this->systemDriver->status(); $allIndices = array_keys($indexStatus['indices']); @@ -419,7 +419,9 @@ public function removeOldIndices() $indicesToBeRemoved[] = $indexName; } - $this->indexManagementDriver->delete($indicesToBeRemoved); + array_map(function ($index) { + $this->indexDriver->deleteIndex($index); + }, $indicesToBeRemoved); return $indicesToBeRemoved; } diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml index 01df676c..4fb466c3 100644 --- a/Configuration/Objects.yaml +++ b/Configuration/Objects.yaml @@ -13,7 +13,7 @@ Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface: factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' factoryMethodName: createIndexerDriver -Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexManagementDriverInterface: +Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexDriverInterface: scope: singleton factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' factoryMethodName: createIndexManagementDriver diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 47adb212..27ddc44a 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -44,7 +44,7 @@ Flowpack: indexer: className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexerDriver' indexManagement: - className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexManagementDriver' + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\IndexDriver' request: className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\RequestDriver' system: @@ -63,7 +63,7 @@ Flowpack: indexer: className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexerDriver' indexManagement: - className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexManagementDriver' + className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\IndexDriver' request: className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2\RequestDriver' system: From 1048dfdef3603e0264001632c2c63acaf76b2d70 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Sat, 4 Feb 2017 22:06:15 +0100 Subject: [PATCH 22/41] TASK: Fix Functional Tests --- .travis.yml | 2 +- .../Testing/ElasticVersion1/Settings.yaml | 5 ++++ .../Testing/ElasticVersion2/Settings.yaml | 5 ++++ Configuration/Testing/Settings.yaml | 2 +- .../Functional/Eel/ElasticSearchQueryTest.php | 25 +++++++++++++++---- 5 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 Configuration/Testing/ElasticVersion1/Settings.yaml create mode 100644 Configuration/Testing/ElasticVersion2/Settings.yaml diff --git a/.travis.yml b/.travis.yml index 642f82e9..f3c82a34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,4 +24,4 @@ install: - cd neos-base-distribution script: - bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit - - bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional + - FLOW_CONTEXT=Testing/ElasticVersion1 bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional diff --git a/Configuration/Testing/ElasticVersion1/Settings.yaml b/Configuration/Testing/ElasticVersion1/Settings.yaml new file mode 100644 index 00000000..c81a7e24 --- /dev/null +++ b/Configuration/Testing/ElasticVersion1/Settings.yaml @@ -0,0 +1,5 @@ +Flowpack: + ElasticSearch: + ContentRepositoryAdaptor: + driver: + version: '1.x' \ No newline at end of file diff --git a/Configuration/Testing/ElasticVersion2/Settings.yaml b/Configuration/Testing/ElasticVersion2/Settings.yaml new file mode 100644 index 00000000..ca0ba0c4 --- /dev/null +++ b/Configuration/Testing/ElasticVersion2/Settings.yaml @@ -0,0 +1,5 @@ +Flowpack: + ElasticSearch: + ContentRepositoryAdaptor: + driver: + version: '2.x' \ No newline at end of file diff --git a/Configuration/Testing/Settings.yaml b/Configuration/Testing/Settings.yaml index 26eae641..553f3515 100644 --- a/Configuration/Testing/Settings.yaml +++ b/Configuration/Testing/Settings.yaml @@ -8,4 +8,4 @@ TYPO3: fileBackend: logFileURL: '%FLOW_PATH_DATA%Logs/ElasticSearch_Testing.log' realtimeIndexing: - enabled: false + enabled: false \ No newline at end of file diff --git a/Tests/Functional/Eel/ElasticSearchQueryTest.php b/Tests/Functional/Eel/ElasticSearchQueryTest.php index 7ef17530..aa2f4eee 100644 --- a/Tests/Functional/Eel/ElasticSearchQueryTest.php +++ b/Tests/Functional/Eel/ElasticSearchQueryTest.php @@ -15,6 +15,7 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\ElasticSearchQueryBuilder; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\ElasticSearchQueryResult; use TYPO3\Flow\Persistence\QueryResultInterface; +use TYPO3\Flow\Tests\FunctionalTestCase; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\TYPO3CR\Domain\Model\Workspace; use TYPO3\TYPO3CR\Domain\Repository\NodeDataRepository; @@ -22,11 +23,9 @@ use TYPO3\TYPO3CR\Domain\Service\Context; use TYPO3\TYPO3CR\Domain\Service\ContextFactoryInterface; use TYPO3\TYPO3CR\Domain\Service\NodeTypeManager; +use TYPO3\TYPO3CR\Search\Search\QueryBuilderInterface; -/** - * Testcase for ElasticSearchQuery - */ -class ElasticSearchQueryTest extends \TYPO3\Flow\Tests\FunctionalTestCase +class ElasticSearchQueryTest extends FunctionalTestCase { /** * @var WorkspaceRepository @@ -106,7 +105,23 @@ public function tearDown() } /** - * @return ElasticSearchQueryBuilder + * @test + */ + public function elasticSearchQueryBuilderStartsClean() + { + /** @var ElasticSearchQueryBuilder $query */ + $query = $this->objectManager->get(ElasticSearchQueryBuilder::class); + $cleanRequestArray = $query->getRequest()->toArray(); + $query->nodeType('TYPO3.Neos.NodeTypes:Page'); + + $query2 = $this->objectManager->get(ElasticSearchQueryBuilder::class); + + $this->assertNotSame($query->getRequest(), $query2->getRequest()); + $this->assertEquals($cleanRequestArray, $query2->getRequest()->toArray()); + } + + /** + * @return QueryBuilderInterface */ protected function getQueryBuilder() { From adabdc06d42c2561118a58c2fdd41156654d5a92 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 8 Feb 2017 10:59:09 +0100 Subject: [PATCH 23/41] TASK: Fix ElasticSearchQueryBuilderTest After refactoring in #71eb4aeda9bd381b2b00fb9142bfe44f36dce9f3 --- .../Eel/ElasticSearchQueryBuilderTest.php | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php index 5f72e348..3575eec7 100644 --- a/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php +++ b/Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php @@ -13,6 +13,7 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Query\FilteredQuery; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\ElasticSearchQueryBuilder; +use TYPO3\Flow\Tests\UnitTestCase; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\TYPO3CR\Domain\Model\Workspace; use TYPO3\TYPO3CR\Domain\Service\Context; @@ -20,7 +21,7 @@ /** * Testcase for ElasticSearchQueryBuilder */ -class ElasticSearchQueryBuilderTest extends \TYPO3\Flow\Tests\UnitTestCase +class ElasticSearchQueryBuilderTest extends UnitTestCase { /** * @var ElasticSearchQueryBuilder @@ -42,7 +43,51 @@ public function setUp() $mockWorkspace->expects($this->any())->method('getName')->will($this->returnValue('user-foo')); $this->queryBuilder = new ElasticSearchQueryBuilder(); - $query = new FilteredQuery($this->queryBuilder); + + $request = [ + 'query' => [ + 'filtered' => [ + 'query' => [ + 'bool' => [ + 'must' => [ + ['match_all' => []] + ] + ] + ], + 'filter' => [ + 'bool' => [ + 'must' => [], + 'should' => [], + 'must_not' => [ + [ + 'term' => ['_hidden' => true] + ], + [ + 'range' => [ + '_hiddenBeforeDateTime' => [ + 'gt' => 'now' + ] + ] + ], + [ + 'range' => [ + '_hiddenAfterDateTime' => [ + 'lt' => 'now' + ] + ] + ] + ] + ] + ] + ] + ], + 'fields' => ['__path'] + ]; + $unsupportedFieldsInCountRequest = ['fields', 'sort', 'from', 'size', 'highlight', 'aggs', 'aggregations']; + + $this->inject($this->queryBuilder, 'request', new FilteredQuery($request, $unsupportedFieldsInCountRequest)); + + $query = new FilteredQuery($this->queryBuilder->getRequest()->toArray(), []); $this->inject($this->queryBuilder, 'request', $query); $this->queryBuilder->query($node); } From d4465fc8aea7178ba0b478fdcc319efbd6e1be5a Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 8 Feb 2017 10:59:54 +0100 Subject: [PATCH 24/41] TASK: Build test matrix for ES Version 1/2 vs PHP 5.6/7 --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f3c82a34..17d3621a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,19 @@ language: php matrix: include: - php: 7.0 + env: ES=1 + - php: 7.0 + env: ES=2 + - php: 5.6 + env: ES=1 - php: 5.6 + env: ES=2 sudo: false before_install: - export NEOS_TARGET_VERSION=2.3 - cd .. - - wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.5.zip && unzip elasticsearch-1.7.5.zip - - mv elasticsearch-1.7.5 elasticsearch + - if [ "$ES" = 1 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.5.zip && unzip elasticsearch-1.7.5.zip && mv elasticsearch-1.7.5 elasticsearch; fi + - if [ "$ES" = 2 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.4.3.zip && unzip elasticsearch-2.4.3.zip && mv elasticsearch-2.4.3 elasticsearch; fi - cd elasticsearch - bin/elasticsearch -d - cd .. @@ -24,4 +30,4 @@ install: - cd neos-base-distribution script: - bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit - - FLOW_CONTEXT=Testing/ElasticVersion1 bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional + - FLOW_CONTEXT="Testing/ElasticVersion{$ES}" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional From 6b25b54bc8c964a089ce0f93157c8131ad2c89eb Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 8 Feb 2017 12:13:34 +0100 Subject: [PATCH 25/41] TASK: Let travis download the patches objectmanager Needs to be removed as soon as the patched version is released. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 17d3621a..f9f2055c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,10 +24,11 @@ before_install: - composer require typo3/typo3cr-search install: - composer install + - curl https://raw.githubusercontent.com/neos/flow-development-collection/3.3/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php -o Packages/Framework/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php - cd .. - rm -rf neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor - mv Flowpack.ElasticSearch.ContentRepositoryAdaptor neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor - cd neos-base-distribution script: - bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit - - FLOW_CONTEXT="Testing/ElasticVersion{$ES}" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional + - FLOW_CONTEXT="Testing/ElasticVersion$ES" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional From 77287169f5766003234a26901dba3dbb57a3de36 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 8 Feb 2017 14:13:30 +0100 Subject: [PATCH 26/41] TASK: Pin typo3cr-search to use version ~2.0.0 --- .travis.yml | 3 +-- composer.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f9f2055c..fc63feaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,8 +20,7 @@ before_install: - cd .. - git clone https://github.com/neos/neos-base-distribution.git -b ${NEOS_TARGET_VERSION} - cd neos-base-distribution - - composer require flowpack/elasticsearch-contentrepositoryadaptor - - composer require typo3/typo3cr-search + - composer require typo3/typo3cr-search:2.0 flowpack/elasticsearch-contentrepositoryadaptor install: - composer install - curl https://raw.githubusercontent.com/neos/flow-development-collection/3.3/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php -o Packages/Framework/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php diff --git a/composer.json b/composer.json index 3c810e55..2e538475 100644 --- a/composer.json +++ b/composer.json @@ -8,7 +8,7 @@ "php": ">=5.5.0", "flowpack/elasticsearch": "*", - "typo3/typo3cr-search": "*" + "typo3/typo3cr-search": "~2.0.0" }, "autoload": { "psr-0": { From e3f0c972044cd38e9b79d06004ce85f676082ebd Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 8 Feb 2017 14:46:59 +0100 Subject: [PATCH 27/41] TASK: Refactor travis script / use if instead of variables --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc63feaa..8a28f92a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ before_install: - cd .. - git clone https://github.com/neos/neos-base-distribution.git -b ${NEOS_TARGET_VERSION} - cd neos-base-distribution - - composer require typo3/typo3cr-search:2.0 flowpack/elasticsearch-contentrepositoryadaptor + - composer require typo3/typo3cr-search:2.1.2 flowpack/elasticsearch-contentrepositoryadaptor install: - composer install - curl https://raw.githubusercontent.com/neos/flow-development-collection/3.3/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php -o Packages/Framework/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php @@ -30,4 +30,5 @@ install: - cd neos-base-distribution script: - bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit - - FLOW_CONTEXT="Testing/ElasticVersion$ES" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional + - if [ "$ES" = 1 ]; then FLOW_CONTEXT="Testing/ElasticVersion1" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi + - if [ "$ES" = 2 ]; then FLOW_CONTEXT="Testing/ElasticVersion2" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi From f00a430a95e26358dfa5c6e651ac8cee4c39a5cc Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 8 Feb 2017 15:27:06 +0100 Subject: [PATCH 28/41] TASK: Refactor the object factory Separate friver and query factory --- .../AbstractDriverSpecificObjectFactory.php | 68 +++++++++++++++++++ .../Factory/DriverFactory.php | 52 +------------- .../Factory/QueryFactory.php | 36 ++++++++++ Configuration/Objects.yaml | 2 +- 4 files changed, 106 insertions(+), 52 deletions(-) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php new file mode 100644 index 00000000..3ef4c9e7 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php @@ -0,0 +1,68 @@ +driverVersion); + if (trim($this->driverVersion) === '' || !isset($this->mapping[$version][$type]['className'])) { + throw new DriverConfigurationException(sprintf('Missing or wrongly configured driver type "%s" with the given version: %s', $type, $version ?: '[missing]'), 1485933538); + } + + $className = trim($this->mapping[$version][$type]['className']); + + $this->logger->log(sprintf('Load %s implementation for Elastic %s (%s)', $type, $version, $className), LOG_DEBUG); + + if (!isset($this->mapping[$version][$type]['arguments'])) { + return new $className(); + } + return new $className(...array_values($this->mapping[$version][$type]['arguments'])); + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php index d1651167..ae211d0f 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php @@ -20,38 +20,10 @@ use TYPO3\Flow\Annotations as Flow; /** - * A factory for creating Elastic Drivers - * * @Flow\Scope("singleton") */ -class DriverFactory +class DriverFactory extends AbstractDriverSpecificObjectFactory { - /** - * @Flow\Inject - * @var LoggerInterface - */ - protected $logger; - - /** - * @var array - * @Flow\InjectConfiguration(path="driver.mapping") - */ - protected $mapping; - - /** - * @var int - * @Flow\InjectConfiguration(path="driver.version") - */ - protected $driverVersion; - - /** - * @return QueryInterface - */ - public function createQuery() - { - return $this->resolve('query'); - } - /** * @return DocumentDriverInterface */ @@ -91,26 +63,4 @@ public function createSystemDriver() { return $this->resolve('system'); } - - /** - * @param string $type - * @return mixed - * @throws DriverConfigurationException - */ - protected function resolve($type) - { - $version = trim($this->driverVersion); - if (trim($this->driverVersion) === '' || !isset($this->mapping[$version][$type]['className'])) { - throw new DriverConfigurationException(sprintf('Missing or wrongly configured driver type "%s" with the given version: %s', $type, $version ?: '[missing]'), 1485933538); - } - - $className = trim($this->mapping[$version][$type]['className']); - - $this->logger->log(sprintf('Load %s implementation for Elastic %s (%s)', $type, $version, $className), LOG_DEBUG); - - if (!isset($this->mapping[$version][$type]['arguments'])) { - return new $className(); - } - return new $className(...array_values($this->mapping[$version][$type]['arguments'])); - } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php new file mode 100644 index 00000000..ceaac964 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php @@ -0,0 +1,36 @@ +resolve('query'); + } +} diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml index 4fb466c3..d4d6d37d 100644 --- a/Configuration/Objects.yaml +++ b/Configuration/Objects.yaml @@ -1,6 +1,6 @@ Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface: scope: prototype - factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' + factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\QueryFactory' factoryMethodName: createQuery Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface: From 2955a67b685bafddf438e2a9b26e499d2fc9087d Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 10 Feb 2017 22:08:52 +0100 Subject: [PATCH 29/41] TASK: Set correct dependencies to flowpack/elasticsearch and typo3/typo3cr-search --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 2e538475..20088d37 100644 --- a/composer.json +++ b/composer.json @@ -7,8 +7,8 @@ "require": { "php": ">=5.5.0", - "flowpack/elasticsearch": "*", - "typo3/typo3cr-search": "~2.0.0" + "flowpack/elasticsearch": "^1.1.0", + "typo3/typo3cr-search": "^2.1.0" }, "autoload": { "psr-0": { From ac669a1439e5509d4197f86c5543e144f80bee64 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 10 Feb 2017 22:34:18 +0100 Subject: [PATCH 30/41] TASK: Default driver must support Elastic 1.x --- Configuration/Settings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 27ddc44a..ddffccde 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -2,7 +2,7 @@ Flowpack: ElasticSearch: ContentRepositoryAdaptor: driver: - version: '2.x' + version: '1.x' mapping: '1.x': query: From 7ad20a03beaca2c113124e1146b23620ea2f10aa Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 10 Feb 2017 22:34:46 +0100 Subject: [PATCH 31/41] TASK: Cleanup use statement in QueryFactory --- .../ContentRepositoryAdaptor/Factory/QueryFactory.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php index ceaac964..bff15c0b 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/QueryFactory.php @@ -11,12 +11,7 @@ * source code. */ -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\DriverConfigurationException; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; use TYPO3\Flow\Annotations as Flow; /** From 8b5c0eff725f514cdf4135dd4556e3431323ff7b Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 10 Feb 2017 22:55:45 +0100 Subject: [PATCH 32/41] TASK: Add a FunctionScoreQuery as alternative Query implementation --- .../Version1/Query/FunctionScoreQuery.php | 114 ++++++++++++++++++ .../Version2/Query/FunctionScoreQuery.php | 18 +++ 2 files changed, 132 insertions(+) create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php create mode 100644 Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FunctionScoreQuery.php diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php new file mode 100644 index 00000000..6f24a390 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FunctionScoreQuery.php @@ -0,0 +1,114 @@ + [] + ]; + + /** + * @param array $functions + * @return void + */ + public function functions(array $functions) + { + if (isset($functions['functions'])) { + $this->functionScoreRequest = $functions; + } else { + $this->functionScoreRequest['functions'] = $functions; + } + } + + /** + * @param string $scoreMode + * @return void + * @throws Exception\QueryBuildingException + */ + public function scoreMode($scoreMode) + { + if (!in_array($scoreMode, ['multiply', 'first', 'sum', 'avg', 'max', 'min'])) { + throw new Exception\QueryBuildingException('Invalid score mode', 1454016230); + } + $this->functionScoreRequest['score_mode'] = $scoreMode; + } + + /** + * @param string $boostMode + * @return void + * @throws Exception\QueryBuildingException + */ + public function boostMode($boostMode) + { + if (!in_array($boostMode, ['multiply', 'replace', 'sum', 'avg', 'max', 'min'])) { + throw new Exception\QueryBuildingException('Invalid boost mode', 1454016229); + } + $this->functionScoreRequest['boost_mode'] = $boostMode; + } + + /** + * @param integer|float $boost + * @return void + * @throws Exception\QueryBuildingException + */ + public function maxBoost($boost) + { + if (!is_numeric($boost)) { + throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); + } + $this->functionScoreRequest['max_boost'] = $boost; + } + + /** + * @param integer|float $score + * @return void + * @throws Exception\QueryBuildingException + */ + public function minScore($score) + { + if (!is_numeric($score)) { + throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); + } + $this->functionScoreRequest['min_score'] = $score; + } + + /** + * {@inheritdoc} + */ + protected function prepareRequest() + { + if ($this->functionScoreRequest['functions'] === []) { + return parent::prepareRequest(); + } + $currentQuery = $this->request['query']; + + $baseQuery = $this->request; + unset($baseQuery['query']); + + $functionScore = $this->functionScoreRequest; + $functionScore['query'] = $currentQuery; + $query = Arrays::arrayMergeRecursiveOverrule($baseQuery, [ + 'query' => [ + 'function_score' => $functionScore + ] + ]); + + return $query; + } +} diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FunctionScoreQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FunctionScoreQuery.php new file mode 100644 index 00000000..861275f0 --- /dev/null +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/Query/FunctionScoreQuery.php @@ -0,0 +1,18 @@ + Date: Fri, 10 Feb 2017 22:56:14 +0100 Subject: [PATCH 33/41] TASK: QueryBuilder must act as a proxy for the method of the Request object --- .../Eel/ElasticSearchQueryBuilder.php | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php index 640a8901..1f44ccb3 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php @@ -13,6 +13,7 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\QueryBuildingException; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; use TYPO3\Eel\ProtectedContextAwareInterface; @@ -729,4 +730,22 @@ protected function convertHitsToNodes(array $hits) return array_values($nodes); } + + /** + * Proxy method to access the public method of the Request object + * + * This is useful with custom Request type where not wrapper method exist in the QueryBuilder. + * + * @param string $method + * @param array $args + * @return $this + * @throws Exception + */ + public function __call($method, array $args) { + if (!method_exists($this->request, $method)) { + throw new Exception(sprintf('Method "%s" does not exist in the current Request object "%s"', $method, get_class($this->request)), 1486763515); + } + call_user_func_array([$this->request, $method], $args); + return $this; + } } From 5e2bd3324218cc8ae1c2260c61de3d42fe670c18 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 10 Feb 2017 23:29:30 +0100 Subject: [PATCH 34/41] TASK: Add missing check for removed and hidden nodes in Driver 2.x --- .../Driver/Version2/IndexerDriver.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php index 0505af0c..d2fb5f88 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php @@ -127,8 +127,10 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targe ctx._source.__fulltextParts = new HashMap(); } - if (fulltext.size() == 0) { - ctx._source.__fulltextParts.remove(identifier); + if (nodeIsRemoved || nodeIsHidden || fulltext.size() == 0) { + if (ctx._source.__fulltextParts.containsKey(identifier)) { + ctx._source.__fulltextParts.remove(identifier); + } } else { ctx._source.__fulltextParts.put(identifier, fulltext); } From e5281f5d78f9dc95358bbbb18019af6f418a11c0 Mon Sep 17 00:00:00 2001 From: Dominique Feyer Date: Fri, 10 Feb 2017 23:30:33 +0100 Subject: [PATCH 35/41] TASK: Fix CGL issues --- .../ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php index 1f44ccb3..601e4f9e 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php @@ -741,7 +741,8 @@ protected function convertHitsToNodes(array $hits) * @return $this * @throws Exception */ - public function __call($method, array $args) { + public function __call($method, array $args) + { if (!method_exists($this->request, $method)) { throw new Exception(sprintf('Method "%s" does not exist in the current Request object "%s"', $method, get_class($this->request)), 1486763515); } From 38f693100cf8ebae81d5c8c067e4783816748398 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 15 Feb 2017 11:17:17 +0100 Subject: [PATCH 36/41] TASK: Remove temporary ObjectManager fix --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8a28f92a..4dd8d2f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,6 @@ before_install: - composer require typo3/typo3cr-search:2.1.2 flowpack/elasticsearch-contentrepositoryadaptor install: - composer install - - curl https://raw.githubusercontent.com/neos/flow-development-collection/3.3/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php -o Packages/Framework/TYPO3.Flow/Classes/TYPO3/Flow/Object/ObjectManager.php - cd .. - rm -rf neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor - mv Flowpack.ElasticSearch.ContentRepositoryAdaptor neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor From efed6e299254eebeeb4dac64206171bc82d29666 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Thu, 16 Feb 2017 10:00:17 +0100 Subject: [PATCH 37/41] TASK: Changes according to review --- .travis.yml | 2 +- .../Driver/AbstractDriver.php | 2 +- .../Driver/AbstractIndexerDriver.php | 2 +- .../Driver/AbstractQuery.php | 4 ++-- .../Driver/DocumentDriverInterface.php | 3 ++- .../Driver/IndexDriverInterface.php | 2 +- .../Driver/IndexerDriverInterface.php | 1 + .../Driver/QueryInterface.php | 17 ++++++++++++++--- .../Driver/RequestDriverInterface.php | 2 +- .../Driver/Version1/DocumentDriver.php | 2 +- .../Driver/Version1/IndexDriver.php | 2 +- .../Driver/Version1/IndexerDriver.php | 2 +- .../Driver/Version1/Query/FilteredQuery.php | 4 ++-- .../Driver/Version1/RequestDriver.php | 2 +- .../Driver/Version1/SystemDriver.php | 2 +- .../Driver/Version2/DocumentDriver.php | 2 +- .../Driver/Version2/IndexDriver.php | 2 +- .../Driver/Version2/IndexerDriver.php | 2 +- .../Driver/Version2/RequestDriver.php | 2 +- .../Driver/Version2/SystemDriver.php | 2 +- .../Eel/ElasticSearchQueryBuilder.php | 12 ++++++------ 21 files changed, 42 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4dd8d2f8..2db1dd1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ before_install: - cd .. - git clone https://github.com/neos/neos-base-distribution.git -b ${NEOS_TARGET_VERSION} - cd neos-base-distribution - - composer require typo3/typo3cr-search:2.1.2 flowpack/elasticsearch-contentrepositoryadaptor + - composer require neos/content-repository-search:~2.1.0 flowpack/elasticsearch-contentrepositoryadaptor install: - composer install - cd .. diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php index 4bd6d8fc..339638fc 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php @@ -17,7 +17,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Abstract Fulltext Indexer + * Abstract Elasticsearch driver */ abstract class AbstractDriver { diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php index e0ce3d68..0cd0b2e6 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php @@ -17,7 +17,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Abstract Fulltext Indexer + * Abstract Fulltext Indexer Driver */ abstract class AbstractIndexerDriver extends AbstractDriver { diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php index 98283ff2..16f40de5 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractQuery.php @@ -16,7 +16,7 @@ use TYPO3\Flow\Utility\Arrays; /** - * Default Filtered Query + * Abstract Elasticsearch Query */ abstract class AbstractQuery implements QueryInterface, \JsonSerializable, \ArrayAccess, ProtectedContextAwareInterface { @@ -77,7 +77,7 @@ public function toArray() /** * {@inheritdoc} */ - public function getRequestAsJSON() + public function getRequestAsJson() { return json_encode($this); } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php index fbfab6b5..6a26b0aa 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/DocumentDriverInterface.php @@ -10,12 +10,13 @@ * information, please view the LICENSE file which was distributed with this * source code. */ + use Flowpack\ElasticSearch\Domain\Model\Index; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\TYPO3CR\Domain\Model\NodeType; /** - * Document Driver Interface + * Elasticsearch Document Driver Interface */ interface DocumentDriverInterface { diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexDriverInterface.php index a19fbaab..c0c4ff16 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexDriverInterface.php @@ -12,7 +12,7 @@ */ /** - * Elastic Index Driver Interface + * Elasticsearch Index Driver Interface */ interface IndexDriverInterface { diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexerDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexerDriverInterface.php index ae9960c1..715e62a6 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexerDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/IndexerDriverInterface.php @@ -10,6 +10,7 @@ * information, please view the LICENSE file which was distributed with this * source code. */ + use Flowpack\ElasticSearch\Domain\Model\Document as ElasticSearchDocument; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/QueryInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/QueryInterface.php index 84e93f4a..97b0635d 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/QueryInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/QueryInterface.php @@ -30,7 +30,7 @@ public function toArray(); * * @return string */ - public function getRequestAsJSON(); + public function getRequestAsJson(); /** * Get the current count request as JSON string @@ -39,12 +39,13 @@ public function getRequestAsJSON(); * * @return string */ - public function getCountRequestAsJSON(); + public function getCountRequestAsJson(); /** * Add a sort filter to the request * * @param array $configuration + * @return void * @api */ public function addSortFilter($configuration); @@ -53,6 +54,7 @@ public function addSortFilter($configuration); * Set the size (limit) of the request * * @param integer $size + * @return void * @api */ public function size($size); @@ -61,6 +63,7 @@ public function size($size); * Set the from (offset) of the request * * @param integer $size + * @return void * @api */ public function from($size); @@ -69,6 +72,7 @@ public function from($size); * Match the searchword against the fulltext index * * @param string $searchWord + * @return void * @api */ public function fulltext($searchWord); @@ -79,6 +83,7 @@ public function fulltext($searchWord); * * @param integer|boolean $fragmentSize The result fragment size for highlight snippets. If this parameter is FALSE, highlighting will be disabled. * @param integer $fragmentCount The number of highlight fragments to show. + * @return void * @api */ public function highlight($fragmentSize, $fragmentCount = null); @@ -88,7 +93,8 @@ public function highlight($fragmentSize, $fragmentCount = null); * * @param string $name * @param array $aggregationDefinition - * @param null $parentPath + * @param string $parentPath + * @return void * @api * @throws Exception\QueryBuildingException */ @@ -99,6 +105,7 @@ public function aggregation($name, array $aggregationDefinition, $parentPath = n * * @param string $name * @param array $suggestionDefinition + * @return void * @api */ public function suggestions($name, array $suggestionDefinition); @@ -109,6 +116,7 @@ public function suggestions($name, array $suggestionDefinition); * @param string $filterType * @param mixed $filterOptions * @param string $clauseType one of must, should, must_not + * @return void * @throws Exception\QueryBuildingException * @api */ @@ -117,6 +125,7 @@ public function queryFilter($filterType, $filterOptions, $clauseType = 'must'); /** * @param string $path * @param string $value + * @return void */ public function setValueByPath($path, $value); @@ -127,6 +136,7 @@ public function setValueByPath($path, $value); * * @param string $path * @param array $data + * @return void * @throws Exception\QueryBuildingException */ public function appendAtPath($path, array $data); @@ -143,6 +153,7 @@ public function setByPath($path, $requestPart); /** * @param array $request + * @return void */ public function replaceRequest(array $request); } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/RequestDriverInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/RequestDriverInterface.php index 996d1caa..39866191 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/RequestDriverInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/RequestDriverInterface.php @@ -13,7 +13,7 @@ use Flowpack\ElasticSearch\Domain\Model\Index; /** - * Request Driver Interface + * Elasticsearch Request Driver Interface */ interface RequestDriverInterface { diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php index 5864cc67..cc1c1e02 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/DocumentDriver.php @@ -21,7 +21,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeType; /** - * Document Driver for Elastic version 1.x + * Document driver for Elasticsearch version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexDriver.php index b7b64d87..cfda8493 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexDriver.php @@ -18,7 +18,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Index Management Driver for Elastic version 1.x + * Index Management driver for Elasticsearch version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php index 060a550b..fc7920e0 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php @@ -21,7 +21,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Indexer Driver for Elastic version 1.x + * Indexer driver for Elasticsearch version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php index f62f5ba8..a063d6cc 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/Query/FilteredQuery.php @@ -22,7 +22,7 @@ class FilteredQuery extends AbstractQuery /** * {@inheritdoc} */ - public function getCountRequestAsJSON() + public function getCountRequestAsJson() { $request = $this->request; foreach ($this->unsupportedFieldsInCountRequest as $field) { @@ -71,6 +71,6 @@ public function queryFilter($filterType, $filterOptions, $clauseType = 'must') throw new Exception\QueryBuildingException('The given clause type "' . $clauseType . '" is not supported. Must be one of "must", "should", "must_not".', 1383716082); } - return $this->appendAtPath('query.filtered.filter.bool.' . $clauseType, [$filterType => $filterOptions]); + $this->appendAtPath('query.filtered.filter.bool.' . $clauseType, [$filterType => $filterOptions]); } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php index f95a0b05..fd92e247 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/RequestDriver.php @@ -17,7 +17,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Request Driver for Elastic version 1.x + * Request driver for Elasticsearch version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php index 8d27fd90..00b714ba 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/SystemDriver.php @@ -16,7 +16,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * System Driver for Elastic version 1.x + * System driver for Elasticsearch version 1.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php index 14c690dc..c07c67f2 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php @@ -19,7 +19,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeType; /** - * Document Driver for Elastic version 2.x + * Document driver for Elasticsearch version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexDriver.php index 8394ed9a..1aab5241 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexDriver.php @@ -15,7 +15,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Index Management Driver for Elastic version 2.x + * Index management driver for Elasticsearch version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php index d2fb5f88..f7eab10a 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php @@ -18,7 +18,7 @@ use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** - * Indexer Driver for Elastic version 2.x + * Indexer driver for Elasticsearch version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php index c4736e64..4d397339 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/RequestDriver.php @@ -15,7 +15,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * Request Driver for Elastic version 2.x + * Request driver for Elasticsearch version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php index c5a6bf12..d0c83d44 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/SystemDriver.php @@ -15,7 +15,7 @@ use TYPO3\Flow\Annotations as Flow; /** - * System Driver for Elastic version 2.x + * System driver for Elasticsearch version 2.x * * @Flow\Scope("singleton") */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php index 601e4f9e..25d7eead 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php @@ -544,7 +544,7 @@ public function getFullElasticSearchHitForNode(NodeInterface $node) public function fetch() { $timeBefore = microtime(true); - $request = $this->request->getRequestAsJSON(); + $request = $this->request->getRequestAsJson(); $response = $this->elasticSearchClient->getIndex()->request('GET', '/_search', [], $request); $timeAfterwards = microtime(true); @@ -585,7 +585,7 @@ public function execute() public function count() { $timeBefore = microtime(true); - $request = $this->getRequest()->getCountRequestAsJSON(); + $request = $this->getRequest()->getCountRequestAsJson(); $response = $this->elasticSearchClient->getIndex()->request('GET', '/_count', [], $request); $timeAfterwards = microtime(true); @@ -734,19 +734,19 @@ protected function convertHitsToNodes(array $hits) /** * Proxy method to access the public method of the Request object * - * This is useful with custom Request type where not wrapper method exist in the QueryBuilder. + * This is used to call a method of a custom Request type where no corresponding wrapper method exist in the QueryBuilder. * * @param string $method - * @param array $args + * @param array $arguments * @return $this * @throws Exception */ - public function __call($method, array $args) + public function __call($method, array $arguments) { if (!method_exists($this->request, $method)) { throw new Exception(sprintf('Method "%s" does not exist in the current Request object "%s"', $method, get_class($this->request)), 1486763515); } - call_user_func_array([$this->request, $method], $args); + call_user_func_array([$this->request, $method], $arguments); return $this; } } From d11662c1b37659f4f7c6ec63f00e97af940472a1 Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Thu, 16 Feb 2017 10:34:22 +0100 Subject: [PATCH 38/41] TASK: Clean up (mostly) use statements --- .../ContentRepositoryAdaptor/Driver/AbstractDriver.php | 1 - .../Driver/AbstractIndexerDriver.php | 3 --- .../Driver/Version1/IndexerDriver.php | 2 +- .../Driver/Version2/DocumentDriver.php | 1 - .../Driver/Version2/IndexerDriver.php | 1 + .../Eel/ElasticSearchQueryBuilder.php | 10 ++++++++-- .../ContentRepositoryAdaptor/Exception.php | 2 -- .../Exception/DriverConfigurationException.php | 1 - .../Exception/QueryBuildingException.php | 1 - .../Factory/AbstractDriverSpecificObjectFactory.php | 4 +--- .../ContentRepositoryAdaptor/Factory/DriverFactory.php | 3 --- .../ContentRepositoryAdaptor/Indexer/NodeIndexer.php | 2 +- .../ContentRepositoryAdaptor/LoggerInterface.php | 2 -- .../Service/IndexWorkspaceTrait.php | 2 +- .../ViewHelpers/GetHitArrayForNodeViewHelper.php | 1 - 15 files changed, 13 insertions(+), 23 deletions(-) diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php index 339638fc..0fe28801 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractDriver.php @@ -14,7 +14,6 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; use TYPO3\Flow\Annotations as Flow; -use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** * Abstract Elasticsearch driver diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php index 0cd0b2e6..82662584 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/AbstractIndexerDriver.php @@ -11,9 +11,6 @@ * source code. */ -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; -use TYPO3\Flow\Annotations as Flow; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; /** diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php index fc7920e0..06a8ba89 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php @@ -11,7 +11,6 @@ * source code. */ -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractDriver; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractIndexerDriver; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; @@ -98,6 +97,7 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targe if ($closestFulltextNode->isRemoved()) { // fulltext root is removed, abort silently... $this->logger->log(sprintf('NodeIndexer (%s): Fulltext root found for %s (%s) not updated, it is removed', $closestFulltextNodeDocumentIdentifier, $node->getPath(), $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); + return null; } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php index c07c67f2..9bf665cb 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/DocumentDriver.php @@ -15,7 +15,6 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder; use Flowpack\ElasticSearch\Domain\Model\Index; use TYPO3\Flow\Annotations as Flow; -use TYPO3\TYPO3CR\Domain\Model\NodeInterface; use TYPO3\TYPO3CR\Domain\Model\NodeType; /** diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php index f7eab10a..b96ee434 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php @@ -99,6 +99,7 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targe if ($closestFulltextNode->isRemoved()) { // fulltext root is removed, abort silently... $this->logger->log(sprintf('NodeIndexer (%s): Fulltext root found for %s (%s) not updated, it is removed', $closestFulltextNodeDocumentIdentifier, $node->getPath(), $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)'); + return null; } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php index 25d7eead..f7843399 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php @@ -84,6 +84,7 @@ class ElasticSearchQueryBuilder implements QueryBuilderInterface, ProtectedConte /** * The ElasticSearch request, as it is being built up. + * * @var QueryInterface * @Flow\Inject */ @@ -172,7 +173,6 @@ public function sort($configuration) * * This algorithm can be re-checked when https://github.com/elasticsearch/elasticsearch/issues/3300 is merged. * - * * @param integer $limit * @return ElasticSearchQueryBuilder * @api @@ -201,7 +201,6 @@ public function limit($limit) /** * output records starting at $from * - * * @param integer $from * @return ElasticSearchQueryBuilder * @api @@ -304,6 +303,7 @@ public function lessThanOrEqual($propertyName, $value) public function queryFilter($filterType, $filterOptions, $clauseType = 'must') { $this->request->queryFilter($filterType, $filterOptions, $clauseType); + return $this; } @@ -320,6 +320,7 @@ public function queryFilter($filterType, $filterOptions, $clauseType = 'must') public function appendAtPath($path, array $data) { $this->request->appendAtPath($path, $data); + return $this; } @@ -410,6 +411,7 @@ public function fieldBasedAggregation($name, $field, $type = "terms", $parentPat public function aggregation($name, array $aggregationDefinition, $parentPath = null) { $this->request->aggregation($name, $aggregationDefinition, $parentPath); + return $this; } @@ -464,6 +466,7 @@ public function termSuggestions($text, $field = '_all', $name = 'suggestions') public function suggestions($name, array $suggestionDefinition) { $this->request->suggestions($name, $suggestionDefinition); + return $this; } @@ -500,6 +503,7 @@ public function getTotalItems() if (isset($this->result['hits']['total'])) { return (int)$this->result['hits']['total']; } + return 0; } @@ -673,6 +677,7 @@ public function query(NodeInterface $contextNode) public function request($path, $requestPart) { $this->request->setByPath($path, $requestPart); + return $this; } @@ -747,6 +752,7 @@ public function __call($method, array $arguments) throw new Exception(sprintf('Method "%s" does not exist in the current Request object "%s"', $method, get_class($this->request)), 1486763515); } call_user_func_array([$this->request, $method], $arguments); + return $this; } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception.php index 66b5abf0..427e8306 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception.php @@ -11,8 +11,6 @@ * source code. */ -use TYPO3\Flow\Annotations as Flow; - class Exception extends \TYPO3\Flow\Exception { } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php index 828c173c..bbc550a2 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/DriverConfigurationException.php @@ -12,7 +12,6 @@ */ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; -use TYPO3\Flow\Annotations as Flow; /** * Elastic Driver Configuration exception diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/QueryBuildingException.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/QueryBuildingException.php index 5b52b767..a4e5db36 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/QueryBuildingException.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Exception/QueryBuildingException.php @@ -12,7 +12,6 @@ */ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception; -use TYPO3\Flow\Annotations as Flow; /** * A Query Building Exception diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php index 3ef4c9e7..eb3b5af1 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/AbstractDriverSpecificObjectFactory.php @@ -11,10 +11,7 @@ * source code. */ -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\DriverConfigurationException; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; use TYPO3\Flow\Annotations as Flow; @@ -63,6 +60,7 @@ protected function resolve($type) if (!isset($this->mapping[$version][$type]['arguments'])) { return new $className(); } + return new $className(...array_values($this->mapping[$version][$type]['arguments'])); } } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php index ae211d0f..73994983 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Factory/DriverFactory.php @@ -14,9 +14,6 @@ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\DriverConfigurationException; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface; use TYPO3\Flow\Annotations as Flow; /** diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php index d27943ce..ff620f74 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php @@ -12,8 +12,8 @@ */ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface; -use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexDriverInterface; +use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\RequestDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\SystemDriverInterface; use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient; diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/LoggerInterface.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/LoggerInterface.php index e51d5db7..5cf87f11 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/LoggerInterface.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/LoggerInterface.php @@ -11,8 +11,6 @@ * source code. */ -use TYPO3\Flow\Annotations as Flow; - /** * Elasticsearch Logger Interface */ diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Service/IndexWorkspaceTrait.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Service/IndexWorkspaceTrait.php index 6534952e..b326ea80 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Service/IndexWorkspaceTrait.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Service/IndexWorkspaceTrait.php @@ -86,7 +86,7 @@ protected function indexWorkspaceWithDimensions($workspaceName, array $dimension $this->nodeFactory->reset(); $context->getFirstLevelNodeCache()->flush(); - + if ($callback !== null) { $callback($workspaceName, $indexedNodes, $dimensions); } diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/ViewHelpers/GetHitArrayForNodeViewHelper.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/ViewHelpers/GetHitArrayForNodeViewHelper.php index 78c5ccc0..a3c7c291 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/ViewHelpers/GetHitArrayForNodeViewHelper.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/ViewHelpers/GetHitArrayForNodeViewHelper.php @@ -12,7 +12,6 @@ */ use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\ElasticSearchQueryResult; -use TYPO3\Flow\Annotations as Flow; use TYPO3\Fluid\Core\ViewHelper\AbstractViewHelper; use TYPO3\TYPO3CR\Domain\Model\NodeInterface; From a35c7e0d0b6cd113c35aef9cf5da2f3ef34e86cb Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Thu, 16 Feb 2017 10:43:31 +0100 Subject: [PATCH 39/41] TASK: Tweak TravisCI setup a bit Adding `--no-update` to the require, since we install in the next step anyway. Also remove one required package, since it is required by the tested package anyway. One less dependency that can go out of sync. --- .travis.yml | 8 ++++++-- composer.json | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2db1dd1a..3993bfac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ language: php matrix: include: + - php: 7.1 + env: ES=1 + - php: 7.1 + env: ES=2 - php: 7.0 env: ES=1 - php: 7.0 @@ -20,9 +24,9 @@ before_install: - cd .. - git clone https://github.com/neos/neos-base-distribution.git -b ${NEOS_TARGET_VERSION} - cd neos-base-distribution - - composer require neos/content-repository-search:~2.1.0 flowpack/elasticsearch-contentrepositoryadaptor + - composer require --no-update --no-interaction flowpack/elasticsearch-contentrepositoryadaptor install: - - composer install + - composer install --no-interaction - cd .. - rm -rf neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor - mv Flowpack.ElasticSearch.ContentRepositoryAdaptor neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor diff --git a/composer.json b/composer.json index 20088d37..01e8a796 100644 --- a/composer.json +++ b/composer.json @@ -1,14 +1,14 @@ { "name": "flowpack/elasticsearch-contentrepositoryadaptor", "type": "typo3-flow-package", - "description": "This package provides functionality for using Elasticsearch on top of TYPO3CR.Search", + "description": "This package provides functionality for using Elasticsearch on top of Neos.ContentRepository.Search", "homepage": "http://flowpack.org/", "license": ["LGPL-3.0"], "require": { "php": ">=5.5.0", "flowpack/elasticsearch": "^1.1.0", - "typo3/typo3cr-search": "^2.1.0" + "neos/content-repository-search": "^2.1.0" }, "autoload": { "psr-0": { From 764451d709857540c81bcd3ecc8292f3b982e389 Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Fri, 17 Feb 2017 17:52:13 +0100 Subject: [PATCH 40/41] TASK: Remove PHP 7.1 again (in .travis.yml, not targeted) --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3993bfac..6469ed74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,6 @@ language: php matrix: include: - - php: 7.1 - env: ES=1 - - php: 7.1 - env: ES=2 - php: 7.0 env: ES=1 - php: 7.0 From 1b79d2b46bf704a132e00641f363ce7a11e5b3bf Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Fri, 17 Feb 2017 18:25:37 +0100 Subject: [PATCH 41/41] TASK: Update documentation for 2.x configuration --- .../Driver/Version1/IndexerDriver.php | 2 +- .../Driver/Version2/IndexerDriver.php | 2 +- Documentation/ElasticConfiguration-1.6-1.7.md | 2 +- Documentation/ElasticConfiguration-2.x.md | 17 +++++++++++++++++ README.md | 9 +++++---- 5 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 Documentation/ElasticConfiguration-2.x.md diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php index 06a8ba89..02635ef5 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version1/IndexerDriver.php @@ -42,7 +42,7 @@ public function document($indexName, NodeInterface $node, ElasticSearchDocument '_id' => $document->getId() ] ], - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/1.7/docs-update.html [ 'script' => ' fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new LinkedHashMap()); diff --git a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php index b96ee434..3797d26c 100644 --- a/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php +++ b/Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Driver/Version2/IndexerDriver.php @@ -42,7 +42,7 @@ public function document($indexName, NodeInterface $node, ElasticSearchDocument '_retry_on_conflict' => 3 ] ], - // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html + // http://www.elasticsearch.org/guide/en/elasticsearch/reference/2.4/docs-update.html [ 'script' => [ 'inline' => ' diff --git a/Documentation/ElasticConfiguration-1.6-1.7.md b/Documentation/ElasticConfiguration-1.6-1.7.md index 2f15a32a..321f71c4 100644 --- a/Documentation/ElasticConfiguration-1.6-1.7.md +++ b/Documentation/ElasticConfiguration-1.6-1.7.md @@ -1,6 +1,6 @@ # Needed Configuration in configuration.yml for Elasticsearch 1.6.x and 1.7.x -In verson 1.6 the `script.disable_dynamic` settings and the Groovy sandbox as such [have been deprecated] +In version 1.6 the `script.disable_dynamic` settings and the Groovy sandbox as such [have been deprecated] (https://www.elastic.co/guide/en/elasticsearch/reference/1.6/modules-scripting.html#enable-dynamic-scripting). You may continue to use the settings for version 1.5.x, but this is what would be the correct configuration for 1.6.x and 1.7.x. diff --git a/Documentation/ElasticConfiguration-2.x.md b/Documentation/ElasticConfiguration-2.x.md new file mode 100644 index 00000000..3df964fb --- /dev/null +++ b/Documentation/ElasticConfiguration-2.x.md @@ -0,0 +1,17 @@ +# Needed Configuration in configuration.yml for Elasticsearch 2.x + +Since version 2.0 the fine-grained script settings are in place as described in the scripting docs +(https://www.elastic.co/guide/en/elasticsearch/reference/2.4/modules-scripting.html#enable-dynamic-scripting). + +``` +# The following settings are absolutely required for the CR adaptor to work +script.inline: true + +# the following settings secure your cluster +cluster.name: [PUT_YOUR_CUSTOM_NAME_HERE] +network.host: 127.0.0.1 + +# the following settings are well-suited for smaller Elasticsearch instances (e.g. as long as you can stay on one host) +index.number_of_shards: 1 +index.number_of_replicas: 0 +``` diff --git a/README.md b/README.md index 24b52762..0b87f08d 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ main functionalities: * finding Nodes in TypoScript / Eel by arbitrary queries * Full-Text Indexing of Pages and other Documents (of course including the full content) -## Elastic version support +## Elastic version support -You can switch the Elastic driver by editing ```Settings.yaml``` +You can switch the Elastic driver by editing ```Settings.yaml``` (```Flowpack.ElasticSearchContentRepositoryAdaptor.driver.version```) with the following value: * ```1.x``` to support Elastic 1.2 to 1.7 -* ```2.x``` to support Elastic 2.x to 1.4 +* ```2.x``` to support Elastic 2.x _Currently the Driver interfaces as not marked as API, and can be changed to adapt to future needs (especially the support of Elastic v5)._ @@ -48,6 +48,7 @@ Finally, run `./flow nodeindex:build`, and add the search plugin to your page. I There is a need, depending on your version of Elasticsearch, to add specific configuration to your Elasticsearch Configuration File `/config/elasticsearch.yml`. +- [ElasticSearch 2.x](Documentation/ElasticConfiguration-2.x.md) - [ElasticSearch 1.6 to 1.7](Documentation/ElasticConfiguration-1.6-1.7.md) - [ElasticSearch 1.4 to 1.5](Documentation/ElasticConfiguration-1.4-1.5.md) - [ElasticSearch 1.3](Documentation/ElasticConfiguration-1.3.md) @@ -680,7 +681,7 @@ in the NodeTypes.yaml. Generally this works by defining the global mapping at `[ ## Change the default Elastic index name -If you need to run serveral (different) neos instances on the same elasticsearch server you will need to change the Configuration/Settings.yaml indexName for each of your project. +If you need to run serveral (different) neos instances on the same elasticsearch server you will need to change the Configuration/Settings.yaml indexName for each of your project. So `./flow nodeindex:build` or `./flow nodeindex:cleanup` won't overwrite your other sites index.