[BUGFIX] Use doctrine DBAL for refindex update

Classes/Command/CleanupCommandController.php

@@ -15,7 +15,6 @@
 
 use Helhum\Typo3Console\Command\Delegation\ReferenceIndexUpdateDelegate;
 use Helhum\Typo3Console\Mvc\Controller\CommandController;
-use Helhum\Typo3Console\Service\Persistence\PersistenceContext;
 use Psr\Log\LoggerInterface;
 use TYPO3\CMS\Core\Log\LogLevel;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -49,13 +48,13 @@ public function updateReferenceIndexCommand($dryRun = false, $verbose = false, $
         $operation = $dryRun ? 'checkReferenceIndex' : 'updateReferenceIndex';
 
         list($errorCount, $recordCount, $processedTables) = $this->persistenceIntegrityService->{$operation}(
-            new PersistenceContext($GLOBALS['TYPO3_DB'], $GLOBALS['TCA']),
             $this->createReferenceIndexDelegateWithOptions($dryRun, $verbose, $showProgress)
         );
 
         if ($errorCount > 0) {
             $this->outputLine('<info>%d errors were ' . ($dryRun ? 'found' : 'fixed') . ', while ' . ($dryRun ? 'checking' : 'updating') . ' reference index for %d records from %d tables.</info>', [$errorCount, $recordCount, count($processedTables)]);
         } else {
+            $this->outputLine();
             $this->outputLine('<info>Index integrity was perfect!</info>');
         }
     }

Classes/Core/Booting/Scripts.php

@@ -231,6 +231,7 @@ public static function provideCleanClassImplementations(ConsoleBootstrap $bootst
             // Register the legacy schema update in case new API does not exist
             // @deprecated since TYPO3 8.x will be removed once TYPO3 7.6 support is removed
             self::registerImplementation(\Helhum\Typo3Console\Database\Schema\SchemaUpdateInterface::class, \Helhum\Typo3Console\Database\Schema\LegacySchemaUpdate::class);
+            self::registerImplementation(\Helhum\Typo3Console\Service\Persistence\PersistenceContextInterface::class, \Helhum\Typo3Console\Service\Persistence\LegacyPersistenceContext::class);
         }
         self::overrideImplementation(\TYPO3\CMS\Extbase\Command\HelpCommandController::class, \Helhum\Typo3Console\Command\HelpCommandController::class);
         self::overrideImplementation(\TYPO3\CMS\Extbase\Mvc\Cli\Command::class, \Helhum\Typo3Console\Mvc\Cli\Command::class);

Classes/Service/Persistence/LegacyPersistenceContext.php

@@ -0,0 +1,131 @@
+<?php
+namespace Helhum\Typo3Console\Service\Persistence;
+
+/*
+ * This file is part of the TYPO3 Console project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read
+ * LICENSE file that was distributed with this source code.
+ *
+ */
+
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Database\DatabaseConnection;
+
+class LegacyPersistenceContext implements PersistenceContextInterface
+{
+    /**
+     * @var DatabaseConnection
+     */
+    protected $databaseConnection;
+
+    /**
+     * @var array
+     */
+    protected $persistenceConfiguration = [];
+
+    public function __construct(DatabaseConnection $databaseConnection = null, array $persistenceConfiguration = [])
+    {
+        $this->databaseConnection = $databaseConnection ?: $GLOBALS['TYPO3_DB'];
+        $this->persistenceConfiguration = $persistenceConfiguration ?: $GLOBALS['TCA'];
+    }
+
+    /**
+     * @param string $tableName
+     * @throws TableDoesNotExistException
+     * @return array
+     */
+    public function getAllRecordsOfTable($tableName)
+    {
+        $existingTableNames = $this->databaseConnection->admin_get_tables();
+        if (empty($existingTableNames[$tableName])) {
+            throw new TableDoesNotExistException(sprintf('Table "%s" exists in $TCA but does not exist in the database. You should run the Database Analyzer in the Install Tool to fix this.', $tableName), 1495562885);
+        }
+        $selectFields = (BackendUtility::isTableWorkspaceEnabled($tableName) ? 'uid,t3ver_wsid' : 'uid');
+        return $this->databaseConnection->exec_SELECTgetRows($selectFields, $tableName, '1=1');
+    }
+
+    /**
+     * @return int
+     */
+    public function countAllRecordsOfAllTables()
+    {
+        $rowsCount = 0;
+        $existingTableNames = $this->databaseConnection->admin_get_tables();
+        foreach ($this->persistenceConfiguration as $tableName => $_) {
+            if (empty($existingTableNames[$tableName])) {
+                continue;
+            }
+            $rowsCount += $this->databaseConnection->exec_SELECTcountRows('uid', $tableName, '1=1');
+        }
+
+        return $rowsCount;
+    }
+
+    /**
+     * @param string $tableName
+     * @param array $recordIds
+     * @throws \InvalidArgumentException
+     * @return int
+     */
+    public function countLostIndexesOfRecordsInTable($tableName, array $recordIds = [])
+    {
+        return $this->databaseConnection->exec_SELECTcountRows(
+            'hash',
+            'sys_refindex',
+            'tablename=' . $this->databaseConnection->fullQuoteStr($tableName, 'sys_refindex')
+            . ' AND recuid NOT IN (' . implode(',', array_map('intval', $recordIds)) . ')'
+        );
+    }
+
+    /**
+     * @param string $tableName
+     * @param array $recordIds
+     * @throws \InvalidArgumentException
+     * @return bool
+     */
+    public function deleteLostIndexesOfRecordsInTable($tableName, array $recordIds = [])
+    {
+        return $this->databaseConnection->exec_DELETEquery(
+            'sys_refindex',
+            'tablename=' . $this->databaseConnection->fullQuoteStr($tableName, 'sys_refindex')
+            . ' AND recuid NOT IN (' . implode(',', array_map('intval', $recordIds)) . ')'
+        );
+    }
+
+    /**
+     * @param array $processedTables
+     * @return int
+     */
+    public function countLostTables(array $processedTables)
+    {
+        return $this->databaseConnection->exec_SELECTcountRows(
+            'hash',
+            'sys_refindex',
+            'tablename NOT IN (' . implode(',', $this->databaseConnection->fullQuoteArray($processedTables, 'sys_refindex')) . ')'
+        );
+    }
+
+    /**
+     * @param array $processedTables
+     */
+    public function deleteLostTables(array $processedTables)
+    {
+        $this->databaseConnection->exec_DELETEquery(
+            'sys_refindex',
+            'tablename NOT IN (' . implode(',', $this->databaseConnection->fullQuoteArray($processedTables, 'sys_refindex')) . ')'
+        );
+    }
+
+    /**
+     * @return array
+     */
+    public function getPersistenceConfiguration()
+    {
+        return $this->persistenceConfiguration;
+    }
+}

Classes/Service/Persistence/PersistenceContext.php

@@ -13,35 +13,189 @@
  *
  */
 
-use TYPO3\CMS\Core\Database\DatabaseConnection;
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\DBALException;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Database\ConnectionPool;
 
-/**
- * Class PersistenceContext
- */
-class PersistenceContext
+class PersistenceContext implements PersistenceContextInterface
 {
     /**
-     * @var DatabaseConnection
+     * @var ConnectionPool
      */
-    protected $databaseConnection;
+    private $connectionPool;
 
     /**
      * @var array
      */
-    protected $persistenceConfiguration = [];
+    private $persistenceConfiguration = [];
+
+    public function __construct(ConnectionPool $connectionPool, array $persistenceConfiguration = [])
+    {
+        $this->connectionPool = $connectionPool;
+        $this->persistenceConfiguration = $persistenceConfiguration ?: $GLOBALS['TCA'];
+    }
+
+    /**
+     * @param string $tableName
+     * @throws TableDoesNotExistException
+     * @return \Traversable
+     */
+    public function getAllRecordsOfTable($tableName)
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable($tableName);
+        $queryBuilder->getRestrictions()->removeAll();
+        if (BackendUtility::isTableWorkspaceEnabled($tableName)) {
+            $queryBuilder->select('uid', 't3ver_wsid');
+        } else {
+            $queryBuilder->select('uid');
+        }
+        try {
+            return $queryBuilder
+                ->from($tableName)
+                ->execute();
+        } catch (DBALException $e) {
+            throw new TableDoesNotExistException(sprintf('Table "%s" exists in $TCA but does not exist in the database. You should run the Database Analyzer in the Install Tool to fix this.', $tableName), 1495562885, $e);
+        }
+    }
+
+    /**
+     * @return int
+     */
+    public function countAllRecordsOfAllTables()
+    {
+        $rowsCount = 0;
+        foreach ($this->persistenceConfiguration as $tableName => $_) {
+            $queryBuilder = $this->connectionPool->getQueryBuilderForTable($tableName);
+            $queryBuilder->getRestrictions()->removeAll();
+            try {
+                $rowsCount += $queryBuilder
+                    ->count('uid')
+                    ->from($tableName)
+                    ->execute()
+                    ->fetchColumn(0);
+            } catch (DBALException $e) {
+                continue;
+            }
+        }
+
+        return $rowsCount;
+    }
+
+    /**
+     * @param string $tableName
+     * @param array $recordIds
+     * @throws TableDoesNotExistException
+     * @return int
+     */
+    public function countLostIndexesOfRecordsInTable($tableName, array $recordIds = [])
+    {
+        // Main query to find lost records
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_refindex');
+        $queryBuilder->getRestrictions()->removeAll();
+        return $queryBuilder
+            ->count('hash')
+            ->from('sys_refindex')
+            ->where(
+                $queryBuilder->expr()->eq(
+                    'tablename',
+                    $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)
+                ),
+                'NOT EXISTS (' . $this->getSubQueryForRecordsInIndex($tableName) . ')'
+            )
+            ->execute()
+            ->fetchColumn(0);
+    }
 
-    public function __construct(DatabaseConnection $databaseConnection, array $persistenceConfiguration)
+    /**
+     * @param string $tableName
+     * @param array $recordIds
+     * @throws \InvalidArgumentException
+     * @return bool
+     */
+    public function deleteLostIndexesOfRecordsInTable($tableName, array $recordIds = [])
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_refindex');
+        return $queryBuilder->delete('sys_refindex')
+            ->where(
+                $queryBuilder->expr()->eq(
+                    'tablename',
+                    $queryBuilder->createNamedParameter($tableName, \PDO::PARAM_STR)
+                ),
+                'NOT EXISTS (' . $this->getSubQueryForRecordsInIndex($tableName) . ')'
+            )
+            ->execute();
+    }
+
+    /**
+     * @param $tableName
+     * @throws TableDoesNotExistException
+     * @return string
+     */
+    private function getSubQueryForRecordsInIndex($tableName)
+    {
+        $refIndexConnectionName = empty($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping']['sys_refindex']) ? ConnectionPool::DEFAULT_CONNECTION_NAME : $GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping']['sys_refindex'];
+        $tableConnectionName = empty($GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName]) ? ConnectionPool::DEFAULT_CONNECTION_NAME : $GLOBALS['TYPO3_CONF_VARS']['DB']['TableMapping'][$tableName];
+
+        // Subselect based queries only work on the same connection
+        if ($refIndexConnectionName !== $tableConnectionName) {
+            throw new TableDoesNotExistException(
+                sprintf(
+                    'Not checking table "%s" for lost indexes, "sys_refindex" table uses a different connection',
+                    $tableName
+                )
+            );
+        }
+
+        // Searching for lost indexes for this table
+        // Build sub-query to find lost records
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable($tableName);
+        $queryBuilder->getRestrictions()->removeAll();
+        $queryBuilder
+            ->select('uid')
+            ->from($tableName, 'sub_' . $tableName)
+            ->where(
+                $queryBuilder->expr()->eq(
+                    'sub_' . $tableName . '.uid',
+                    $queryBuilder->quoteIdentifier('sys_refindex.recuid')
+                )
+            );
+        return $queryBuilder->getSQL();
+    }
+
+    /**
+     * @param array $processedTables
+     * @return int
+     */
+    public function countLostTables(array $processedTables)
     {
-        $this->databaseConnection = $databaseConnection;
-        $this->persistenceConfiguration = $persistenceConfiguration;
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_refindex');
+        $queryBuilder->getRestrictions()->removeAll();
+        return $queryBuilder
+            ->count('hash')
+            ->from('sys_refindex')
+            ->where(
+                $queryBuilder->expr()->notIn(
+                    'tablename',
+                    $queryBuilder->createNamedParameter($processedTables, Connection::PARAM_STR_ARRAY)
+                )
+            )->execute()
+            ->fetchColumn(0);
     }
 
     /**
-     * @return \TYPO3\CMS\Core\Database\DatabaseConnection
+     * @param array $processedTables
      */
-    public function getDatabaseConnection()
+    public function deleteLostTables(array $processedTables)
     {
-        return $this->databaseConnection;
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_refindex');
+        $queryBuilder->delete('sys_refindex')
+            ->where(
+                $queryBuilder->expr()->notIn(
+                    'tablename',
+                    $queryBuilder->createNamedParameter($processedTables, Connection::PARAM_STR_ARRAY)
+                )
+            )->execute();
     }
 
     /**