[CI] Add Rector CI (tomasnorre#486)

.travis.yml

@@ -81,6 +81,7 @@ jobs:
     - env: TYPO3_VERSION=^10.0 COVERAGE=0 FUNCTIONAL=0
     - env: TYPO3_VERSION=dev-master COVERAGE=0 FUNCTIONAL=0
     - env: TYPO3_VERSION=dev-master COVERAGE=0 FUNCTIONAL=1
+    - name: "Rector CI"
   include:
     - stage: test
       env: TYPO3_VERSION=^9.5 COVERAGE=0 FUNCTIONAL=0
@@ -118,6 +119,13 @@ jobs:
       env: TYPO3_VERSION=dev-master COVERAGE=0 FUNCTIONAL=1
       php: 7.3
 
+    # Rector CI
+    - stage: rector
+      name: "Rector CI"
+      php: 7.2
+      script:
+        - .Build/bin/rector process --dry-run
+
     - stage: ship to ter
       if: tag IS present
       php: 7.0

Classes/Api/CrawlerApi.php

@@ -31,7 +31,6 @@
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Database\Query\QueryBuilder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Core\Utility\MathUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 use TYPO3\CMS\Frontend\Page\PageRepository;
 

Classes/Command/BuildQueueCommand.php

@@ -29,10 +29,10 @@
 use AOE\Crawler\Controller\CrawlerController;
 use AOE\Crawler\Domain\Model\Reason;
 use AOE\Crawler\Event\EventDispatcher;
+use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\Console\Command\Command;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\MathUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
@@ -108,23 +108,22 @@ protected function configure(): void
      *
      * --- Re-cache pages from page 7 and two levels down, executed immediately
      * $ typo3 crawler:buildQueue --page 7 --depth 2 --conf defaultConfiguration --mode exec
-
+     *
      *
      * --- Put entries for re-caching pages from page 7 into queue, 4 every minute.
      * $ typo3 crawler:buildQueue --page 7 --depth 0 --conf defaultConfiguration --mode queue --number 4
      *
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-
         $mode = $input->getOption('mode');
 
         $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
 
         /** @var CrawlerController $crawlerController */
         $crawlerController = $objectManager->get(CrawlerController::class);
 
-        if ( $mode === 'exec') {
+        if ($mode === 'exec') {
             $crawlerController->registerQueueEntriesInternallyOnly = true;
         }
 
@@ -166,39 +165,39 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $pageId,
             MathUtility::forceIntegerInRange($input->getOption('depth'), 0, 99),
             $crawlerController->getCurrentTime(),
-            MathUtility::forceIntegerInRange($input->getOption('number') ?:  30, 1, 1000),
+            MathUtility::forceIntegerInRange($input->getOption('number') ?: 30, 1, 1000),
             $mode === 'queue' || $mode === 'exec',
             $mode === 'url',
             [],
             $configurationKeys
         );
 
         if ($mode === 'url') {
-             $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->downloadUrls) . PHP_EOL . '</info>');
+            $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->downloadUrls) . PHP_EOL . '</info>');
         } elseif ($mode === 'exec') {
-             $output->writeln('<info>Executing ' . count($crawlerController->urlList) . ' requests right away:</info>');
-             $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->urlList) . '</info>' . PHP_EOL);
-             $output->writeln('<info>Processing</info>' . PHP_EOL);
+            $output->writeln('<info>Executing ' . count($crawlerController->urlList) . ' requests right away:</info>');
+            $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->urlList) . '</info>' . PHP_EOL);
+            $output->writeln('<info>Processing</info>' . PHP_EOL);
 
             foreach ($crawlerController->queueEntries as $queueRec) {
                 $p = unserialize($queueRec['parameters']);
-                 $output->writeln('<info>' . $p['url'] . ' (' . implode(',', $p['procInstructions']) . ') => ' . '</info>' . PHP_EOL);
+                $output->writeln('<info>' . $p['url'] . ' (' . implode(',', $p['procInstructions']) . ') => ' . '</info>' . PHP_EOL);
                 $result = $crawlerController->readUrlFromArray($queueRec);
 
                 $requestResult = unserialize($result['content']);
                 if (is_array($requestResult)) {
                     $resLog = is_array($requestResult['log']) ? PHP_EOL . chr(9) . chr(9) . implode(PHP_EOL . chr(9) . chr(9), $requestResult['log']) : '';
-                     $output->writeln('<info>OK: ' . $resLog . '</info>' . PHP_EOL);
+                    $output->writeln('<info>OK: ' . $resLog . '</info>' . PHP_EOL);
                 } else {
-                     $output->writeln('<errror>Error checking Crawler Result:  ' . substr(preg_replace('/\s+/', ' ', strip_tags($result['content'])), 0, 30000) . '...' . PHP_EOL . '</errror>' . PHP_EOL);
+                    $output->writeln('<errror>Error checking Crawler Result:  ' . substr(preg_replace('/\s+/', ' ', strip_tags($result['content'])), 0, 30000) . '...' . PHP_EOL . '</errror>' . PHP_EOL);
                 }
             }
         } elseif ($mode === 'queue') {
-             $output->writeln('<info>Putting ' . count($crawlerController->urlList) . ' entries in queue:</info>' . PHP_EOL);
-             $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->urlList) . '</info>' . PHP_EOL);
+            $output->writeln('<info>Putting ' . count($crawlerController->urlList) . ' entries in queue:</info>' . PHP_EOL);
+            $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->urlList) . '</info>' . PHP_EOL);
         } else {
-             $output->writeln('<info>' . count($crawlerController->urlList) . ' entries found for processing. (Use "mode" to decide action):</info>' . PHP_EOL);
-             $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->urlList) . '</info>' . PHP_EOL);
+            $output->writeln('<info>' . count($crawlerController->urlList) . ' entries found for processing. (Use "mode" to decide action):</info>' . PHP_EOL);
+            $output->writeln('<info>' . implode(PHP_EOL, $crawlerController->urlList) . '</info>' . PHP_EOL);
         }
     }
 
@@ -213,6 +212,4 @@ private function getConfigurationKeys($conf)
         $parameter = trim($conf);
         return ($parameter != '' ? GeneralUtility::trimExplode(',', $parameter) : []);
     }
-
-
 }

Classes/Command/FlushQueueCommand.php

@@ -27,10 +27,10 @@
  ***************************************************************/
 
 use AOE\Crawler\Controller\CrawlerController;
+use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\Console\Command\Command;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\MathUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
@@ -58,7 +58,7 @@ protected function configure(): void
             InputOption::VALUE_OPTIONAL,
             'Output mode',
             'finished'
-		);
+        );
 
         $this->addOption(
             'page',
@@ -107,6 +107,4 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 break;
         }
     }
-
-
 }

Classes/Command/ProcessQueueCommand.php

@@ -135,5 +135,4 @@ public function execute(InputInterface $input, OutputInterface $output)
 
         return $output->writeln($result);
     }
-
 }

Classes/Controller/CrawlerController.php

@@ -25,12 +25,12 @@
  *  This copyright notice MUST APPEAR in all copies of the script!
  ***************************************************************/
 
-use AOE\Crawler\QueueExecutor;
 use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
 use AOE\Crawler\Domain\Repository\ConfigurationRepository;
 use AOE\Crawler\Domain\Repository\ProcessRepository;
 use AOE\Crawler\Domain\Repository\QueueRepository;
 use AOE\Crawler\Event\EventDispatcher;
+use AOE\Crawler\QueueExecutor;
 use AOE\Crawler\Utility\SignalSlotUtility;
 use Psr\Http\Message\UriInterface;
 use Psr\Log\LoggerAwareInterface;
@@ -449,7 +449,6 @@ public function urlListFromUrlArray(
         array &$downloadUrls,
         array $incomingProcInstructions
     ) {
-
         if (!is_array($vv['URLs'])) {
             return 'ERROR - no URL generated';
         }
@@ -1359,7 +1358,7 @@ public function readUrl($queueId, $force = false)
      */
     public function readUrlFromArray($field_array)
     {
-            // Set exec_time to lock record:
+        // Set exec_time to lock record:
         $field_array['exec_time'] = $this->getCurrentTime();
         $connectionForCrawlerQueue = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable($this->tableName);
         $connectionForCrawlerQueue->insert(

Classes/CrawlStrategy/GuzzleExecutionStrategy.php

@@ -59,7 +59,7 @@ public function fetchUrlContents(UriInterface $url, string $crawlerId)
                 sprintf('Error while opening "%s" - ' . $e->getResponse()->getStatusCode() . chr(32) . $e->getResponse()->getReasonPhrase(), $url),
                 ['crawlerId' => $crawlerId]
             );
-            return $e->getResponse()->getStatusCode() . chr(32) .  $e->getResponse()->getReasonPhrase();
+            return $e->getResponse()->getStatusCode() . chr(32) . $e->getResponse()->getReasonPhrase();
         }
     }
 
@@ -77,5 +77,4 @@ protected function buildRequestHeaders(string $crawlerId): array
             'User-Agent' => 'TYPO3 crawler'
         ];
     }
-
 }

Classes/Domain/Model/Configuration.php

@@ -243,4 +243,3 @@ public function setExclude($exclude)
         $this->exclude = $exclude;
     }
 }
-

Classes/Domain/Repository/ConfigurationRepository.php

@@ -62,7 +62,6 @@ public function getCrawlerConfigurationRecords(): array
         return $records;
     }
 
-
     /**
      * Traverses up the rootline of a page and fetches all crawler records.
      *

Classes/Domain/Repository/QueueRepository.php

@@ -28,7 +28,6 @@
 use AOE\Crawler\Domain\Model\Process;
 use AOE\Crawler\Domain\Model\Queue;
 use TYPO3\CMS\Core\Database\ConnectionPool;
-use TYPO3\CMS\Core\Database\Query\QueryBuilder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\MathUtility;
 

Classes/Middleware/CrawlerInitialization.php

@@ -15,7 +15,6 @@
  * The TYPO3 project - inspiring people to share!
  */
 
-
 use AOE\Crawler\Domain\Repository\QueueRepository;
 use Psr\Http\Message\ResponseInterface;
 use Psr\Http\Message\ServerRequestInterface;
@@ -142,5 +141,4 @@ protected function runPollSuccessHooks()
             }
         }
     }
-
 }

Classes/QueueExecutor.php

@@ -15,12 +15,11 @@
  * The TYPO3 project - inspiring people to share!
  */
 
-
+use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
+use AOE\Crawler\Controller\CrawlerController;
 use AOE\Crawler\CrawlStrategy\CallbackExecutionStrategy;
 use AOE\Crawler\CrawlStrategy\GuzzleExecutionStrategy;
 use AOE\Crawler\CrawlStrategy\SubProcessExecutionStrategy;
-use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
-use AOE\Crawler\Controller\CrawlerController;
 use AOE\Crawler\Event\EventDispatcher;
 use TYPO3\CMS\Core\Http\Uri;
 use TYPO3\CMS\Core\SingletonInterface;

Classes/Worker/CrawlerWorker.php

@@ -16,8 +16,8 @@
 
 use AOE\Crawler\Controller\CrawlerController;
 use AOEPeople\Crawler\Hooks\IndexedSearchCrawlerFilesHook;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\TimeTracker\TimeTracker;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\IndexedSearch\Worker\WorkerInterface;
 
 class CrawlerWorker implements WorkerInterface
@@ -44,7 +44,7 @@ public function index(object $caller, array $conf, string $file, string $content
 
         $params = [
             'document' => $contentTmpFile,
-            'alturl' =>$file,
+            'alturl' => $file,
             'conf' => $conf
         ];
 
@@ -54,4 +54,4 @@ public function index(object $caller, array $conf, string $file, string $content
 
         GeneralUtility::makeInstance(TimeTracker::class)->setTSlogMessage('media "' . $params['document'] . '" added to "crawler" queue.', 1);
     }
-}
+}

Tests/Functional/Domain/Repository/ProcessRepositoryTest.php

@@ -27,7 +27,6 @@
 
 use AOE\Crawler\Domain\Repository\ProcessRepository;
 use Nimut\TestingFramework\TestCase\FunctionalTestCase;
-use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
 

Tests/Functional/Domain/Repository/QueueRepositoryTest.php

@@ -26,7 +26,6 @@
  ***************************************************************/
 
 use AOE\Crawler\Domain\Model\Process;
-use AOE\Crawler\Domain\Model\Queue;
 use AOE\Crawler\Domain\Repository\QueueRepository;
 use Nimut\TestingFramework\TestCase\FunctionalTestCase;
 use TYPO3\CMS\Core\Utility\GeneralUtility;

Tests/Unit/Controller/CrawlerControllerTest.php

@@ -25,7 +25,6 @@
  *  This copyright notice MUST APPEAR in all copies of the script!
  ***************************************************************/
 
-use AOE\Crawler\Command\QueueCommandLineController;
 use AOE\Crawler\Controller\CrawlerController;
 use Nimut\TestingFramework\TestCase\UnitTestCase;
 use Psr\Log\NullLogger;

composer.json

@@ -16,6 +16,7 @@
         }
     ],
     "minimum-stability": "dev",
+    "prefer-stable": true,
     "require": {
         "php": ">=7.2",
         "ext-json": "*",
@@ -31,7 +32,8 @@
         "nimut/testing-framework": "^4.1",
         "nimut/typo3-complete": "^9.5",
         "friendsofphp/php-cs-fixer": "^2.15",
-        "phpstan/phpstan": "^0.11.4"
+        "phpstan/phpstan": "^0.11.4",
+        "rector/rector": "^0.6.0@dev"
     },
     "replace": {
         "typo3-ter/crawler": "self.version"

rector-ci.yaml

@@ -0,0 +1,20 @@
+parameters:
+    paths:
+        - "Classes"
+        - "Configuration"
+        - "Migrations"
+        - "Tests"
+
+    sets:
+        - "dead-code"
+
+    auto_import_names: false
+
+    autoload_paths:
+        - "Classes"
+
+    exclude_paths:
+        # missing parent interface "TYPO3\CMS\IndexedSearch\Worker\WorkerInterface"
+        - "Classes/Worker/CrawlerWorker.php"
+        # BC layer
+        - "Migrations/Code/LegacyClassesForIde.php"