Permalink
Browse files

Pagination using SQL walkers

A CountSqlWalker and LimitSubquerySqlWalker have been implemented. By
default the Paginator will use these SQL walkers. When a query already
uses custom SQL walkers, the Paginator will fall back to the existing
TreeWalker implementations. Improvements:

* Support for more complex DQL queries using named mixed results with
  GROUP BY and HAVING. For example:

  SELECT g, u, COUNT(u.id) AS userCount
      FROM Entity\Group g LEFT JOIN g.users u
      GROUP BY g.id
      HAVING userCount > 0

* Support for entities with composite primary keys in the CountSqlWalker
  and LimitSubquerySqlWalker. Only the WhereInWalker still needs to be
  updated for full composite primary key support. But someone smarter
  than me needs to look at that and figure out how to build a WHERE IN
  query that can select rows based on multiple columns.
  • Loading branch information...
sandermarechal committed Mar 6, 2012
1 parent f6a61b1 commit edd5d14b06f03187bc512c8c1c066539ee3ceed0
@@ -0,0 +1,120 @@
+<?php
+/**
+ * Doctrine ORM
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to kontakt@beberlei.de so I can send you a copy immediately.
+ */
+
+namespace Doctrine\ORM\Tools\Pagination;
+
+use Doctrine\ORM\Query\SqlWalker,
+ Doctrine\ORM\Query\AST\SelectStatement;
+
+/**
+ * Wrap the query in order to accurately count the root objects
+ *
+ * Given a DQL like `SELECT u FROM User u` it will generate an SQL query like:
+ * SELECT COUNT(*) (SELECT DISTINCT <id> FROM (<original SQL>))
+ *
+ * Works with composite keys but cannot deal with queries that have multiple
+ * root entities (e.g. `SELECT f, b from Foo, Bar`)
+ *
+ * @author Sander Marechal <s.marechal@jejik.com>
+ */
+class CountSqlWalker extends SqlWalker
+{
+ /**
+ * @var Doctrine\DBAL\Platforms\AbstractPlatform
+ */
+ private $platform;
+
+ /**
+ * @var Doctrine\ORM\Query\ResultSetMapping
+ */
+ private $rsm;
+
+ /**
+ * @var array
+ */
+ private $queryComponents;
+
+ /**
+ * Constructor. Stores various parameters that are otherwise unavailable
+ * because Doctrine\ORM\Query\SqlWalker keeps everything private without
+ * accessors.
+ *
+ * @param Doctrine\ORM\Query $query
+ * @param Doctrine\ORM\Query\ParserResult $parserResult
+ * @param array $queryComponents
+ */
+ public function __construct($query, $parserResult, array $queryComponents)
+ {
+ $this->platform = $query->getEntityManager()->getConnection()->getDatabasePlatform();
+ $this->rsm = $parserResult->getResultSetMapping();
+ $this->queryComponents = $queryComponents;
+
+ parent::__construct($query, $parserResult, $queryComponents);
+ }
+
+ /**
+ * Walks down a SelectStatement AST node, wrapping it in a COUNT (SELECT DISTINCT)
+ *
+ * Note that the ORDER BY clause is not removed. Many SQL implementations (e.g. MySQL)
+ * are able to cache subqueries. By keeping the ORDER BY clause intact, the limitSubQuery
+ * that will most likely be executed next can be read from the native SQL cache.
+ *
+ * @param SelectStatement $AST
+ * @return string
+ */
+ public function walkSelectStatement(SelectStatement $AST)
+ {
+ $sql = parent::walkSelectStatement($AST);
+
+ // Find out the SQL alias of the identifier column of the root entity
+ // It may be possible to make this work with multiple root entities but that
+ // would probably require issuing multiple queries or doing a UNION SELECT
+ // so for now, It's not supported.
+
+ // Get the root entity and alias from the AST fromClause
+ $from = $AST->fromClause->identificationVariableDeclarations;
+ if (count($from) > 1) {
+ throw new \RuntimeException("Cannot count query which selects two FROM components, cannot make distinction");
+ }
+
+ $rootClass = $from[0]->rangeVariableDeclaration->abstractSchemaName;
+ $rootAlias = $from[0]->rangeVariableDeclaration->aliasIdentificationVariable;
+
+ // Get the identity properties from the metadata
+ $rootIdentifier = $this->queryComponents[$rootAlias]['metadata']->identifier;
+
+ // For every identifier, find out the SQL alias by combing through the ResultSetMapping
+ $sqlIdentifier = array();
+ foreach ($rootIdentifier as $property) {
+ foreach (array_keys($this->rsm->fieldMappings, $property) as $alias) {
+ if ($this->rsm->columnOwnerMap[$alias] == $rootAlias) {
+ $sqlIdentifier[$property] = $alias;
+ }
+ }
+ }
+
+ if (count($rootIdentifier) != count($sqlIdentifier)) {
+ throw new \RuntimeException(sprintf(
+ 'Not all identifier properties can be found in the ResultSetMapping: %s',
+ implode(', ', array_diff($rootIdentifier, array_keys($sqlIdentifier)))
+ ));
+ }
+
+ // Build the counter query
+ return sprintf('SELECT %s AS _dctrn_count FROM (SELECT DISTINCT %s FROM (%s) AS _dctrn_result) AS _dctrn_table',
+ $this->platform->getCountExpression('*'),
+ implode(', ', $sqlIdentifier),
+ $sql
+ );
+ }
+}
@@ -0,0 +1,144 @@
+<?php
+/**
+ * Doctrine ORM
+ *
+ * LICENSE
+ *
+ * This source file is subject to the new BSD license that is bundled
+ * with this package in the file LICENSE.txt.
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to kontakt@beberlei.de so I can send you a copy immediately.
+ */
+
+namespace Doctrine\ORM\Tools\Pagination;
+
+use Doctrine\ORM\Query\SqlWalker,
+ Doctrine\ORM\Query\AST\SelectStatement;
+
+/**
+ * Wrap the query in order to select root entity IDs for pagination
+ *
+ * Given a DQL like `SELECT u FROM User u` it will generate an SQL query like:
+ * SELECT DISTINCT <id> FROM (<original SQL>) LIMIT x OFFSET y
+ *
+ * Works with composite keys but cannot deal with queries that have multiple
+ * root entities (e.g. `SELECT f, b from Foo, Bar`)
+ *
+ * @author Sander Marechal <s.marechal@jejik.com>
+ */
+class LimitSubquerySqlWalker extends SqlWalker
+{
+ /**
+ * @var Doctrine\DBAL\Platforms\AbstractPlatform
+ */
+ private $platform;
+
+ /**
+ * @var Doctrine\ORM\Query\ResultSetMapping
+ */
+ private $rsm;
+
+ /**
+ * @var array
+ */
+ private $queryComponents;
+
+ /**
+ * @var int
+ */
+ private $firstResult;
+
+ /**
+ * @var int
+ */
+ private $maxResults;
+
+ /**
+ * Constructor. Stores various parameters that are otherwise unavailable
+ * because Doctrine\ORM\Query\SqlWalker keeps everything private without
+ * accessors.
+ *
+ * @param Doctrine\ORM\Query $query
+ * @param Doctrine\ORM\Query\ParserResult $parserResult
+ * @param array $queryComponents
+ */
+ public function __construct($query, $parserResult, array $queryComponents)
+ {
+ $this->platform = $query->getEntityManager()->getConnection()->getDatabasePlatform();
+ $this->rsm = $parserResult->getResultSetMapping();
+ $this->queryComponents = $queryComponents;
+
+ // Reset limit and offset
+ $this->firstResult = $query->getFirstResult();
+ $this->maxResults = $query->getMaxResults();
+ $query->setFirstResult(null)->setMaxResults(null);
+
+ parent::__construct($query, $parserResult, $queryComponents);
+ }
+
+ /**
+ * Walks down a SelectStatement AST node, wrapping it in a SELECT DISTINCT
+ *
+ * @param SelectStatement $AST
+ * @return string
+ */
+ public function walkSelectStatement(SelectStatement $AST)
+ {
+ $sql = parent::walkSelectStatement($AST);
+
+ // Find out the SQL alias of the identifier column of the root entity
+ // It may be possible to make this work with multiple root entities but that
+ // would probably require issuing multiple queries or doing a UNION SELECT
+ // so for now, It's not supported.
+
+ // Get the root entity and alias from the AST fromClause
+ $from = $AST->fromClause->identificationVariableDeclarations;
+ if (count($from) !== 1) {
+ throw new \RuntimeException("Cannot count query which selects two FROM components, cannot make distinction");
+ }
+
+ $rootClass = $from[0]->rangeVariableDeclaration->abstractSchemaName;
+ $rootAlias = $from[0]->rangeVariableDeclaration->aliasIdentificationVariable;
+
+ // Get the identity properties from the metadata
+ $metadata = $this->queryComponents[$rootAlias]['metadata'];
+ $rootIdentifier = $metadata->identifier;
+
+ // For every identifier, find out the SQL alias by combing through the ResultSetMapping
+ $sqlIdentifier = array();
+ foreach ($rootIdentifier as $property) {
+ foreach (array_keys($this->rsm->fieldMappings, $property) as $alias) {
+ if ($this->rsm->columnOwnerMap[$alias] == $rootAlias) {
+ $sqlIdentifier[$property] = $alias;
+ }
+ }
+ }
+
+ if (count($rootIdentifier) != count($sqlIdentifier)) {
+ throw new \RuntimeException(sprintf(
+ 'Not all identifier properties can be found in the ResultSetMapping: %s',
+ implode(', ', array_diff($rootIdentifier, array_keys($sqlIdentifier)))
+ ));
+ }
+
+ // Build the counter query
+ $sql = sprintf('SELECT DISTINCT %s FROM (%s) AS _dctrn_result',
+ implode(', ', $sqlIdentifier), $sql);
+
+ // Apply the limit and offset
+ $sql = $this->platform->modifyLimitQuery(
+ $sql, $this->maxResults, $this->firstResult
+ );
+
+ // Add the columns to the ResultSetMapping. It's not really nice but
+ // it works. Preferably I'd clear the RSM or simply create a new one
+ // but that is not possible from inside the output walker, so we dirty
+ // up the one we have.
+ foreach ($sqlIdentifier as $property => $alias) {
+ $this->rsm->addScalarResult($alias, $property);
+ }
+
+ return $sql;
+ }
+}
@@ -21,9 +21,12 @@
use Doctrine\ORM\QueryBuilder;
use Doctrine\ORM\Query;
+use Doctrine\ORM\Query\ResultSetMapping;
use Doctrine\ORM\NoResultException;
use Doctrine\ORM\Tools\Pagination\WhereInWalker;
+use Doctrine\ORM\Tools\Pagination\WhereInSqlWalker;
use Doctrine\ORM\Tools\Pagination\CountWalker;
+use Doctrine\ORM\Tools\Pagination\CountSqlWalker;
use Countable;
use IteratorAggregate;
use ArrayIterator;
@@ -49,6 +52,11 @@ class Paginator implements \Countable, \IteratorAggregate
*/
private $fetchJoinCollection;
+ /**
+ * @var bool|null
+ */
+ private $useSqlWalkers;
+
/**
* @var int
*/
@@ -90,6 +98,28 @@ public function getFetchJoinCollection()
return $this->fetchJoinCollection;
}
+ /**
+ * Returns whether the paginator will use an SQL TreeWalker
+ *
+ * @return bool|null
+ */
+ public function getUseSqlWalkers()
+ {
+ return $this->useSqlWalkers;
+ }
+
+ /**
+ * Set whether the paginator will use an SQL TreeWalker
+ *
+ * @param bool|null $useSqlWalkers
+ * @return $this
+ */
+ public function setUseSqlWalkers($useSqlWalkers)
+ {
+ $this->useSqlWalkers = $useSqlWalkers;
+ return $this;
+ }
+
/**
* {@inheritdoc}
*/
@@ -103,7 +133,16 @@ public function count()
$countQuery->setHint(CountWalker::HINT_DISTINCT, true);
}
- $countQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\CountWalker'));
+ if ($this->useSqlWalker($countQuery)) {
+ $rsm = new ResultSetMapping();
+ $rsm->addScalarResult('_dctrn_count', 'count');
+
+ $countQuery->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\CountSqlWalker');
+ $countQuery->setResultSetMapping($rsm);
+ } else {
+ $countQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\CountWalker'));
+ }
+
$countQuery->setFirstResult(null)->setMaxResults(null);
try {
@@ -127,9 +166,14 @@ public function getIterator()
if ($this->fetchJoinCollection) {
$subQuery = $this->cloneQuery($this->query);
- $subQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\LimitSubqueryWalker'))
- ->setFirstResult($offset)
- ->setMaxResults($length);
+
+ if ($this->useSqlWalker($subQuery)) {
+ $subQuery->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\LimitSubquerySqlWalker');
+ } else {
+ $subQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\LimitSubqueryWalker'));
+ }
+
+ $subQuery->setFirstResult($offset)->setMaxResults($length);
$ids = array_map('current', $subQuery->getScalarResult());
@@ -176,5 +220,21 @@ private function cloneQuery(Query $query)
return $cloneQuery;
}
+
+ /**
+ * Determine whether to use an SQL TreeWalker for the query
+ *
+ * @param Query $query The query.
+ *
+ * @return bool
+ */
+ private function useSqlWalker(Query $query)
+ {
+ if ($this->useSqlWalkers === null) {
+ return (Boolean) $query->getHint(Query::HINT_CUSTOM_OUTPUT_WALKER) == false;
+ }
+
+ return $this->useSqlWalkers;
+ }
}
Oops, something went wrong.

0 comments on commit edd5d14

Please sign in to comment.