[FEATURE] Upgrade Automation

.travis.yml

@@ -15,14 +15,14 @@ matrix:
       env: TYPO3_VERSION=^7.6
     - php: 7.0
       env: TYPO3_VERSION=^7.6
-    - php: 7.0
-      env: TYPO3_VERSION=^8.7
-    - php: 7.0
-      env: TYPO3_VERSION="dev-master as 8.7.0"
     - php: 7.1
       env: TYPO3_VERSION=^7.6
+    - php: 7.0
+      env: TYPO3_VERSION=^8.7
     - php: 7.1
       env: TYPO3_VERSION=^8.7
+    - php: 7.0
+      env: TYPO3_VERSION="dev-master as 8.7.0"
     - php: 7.1
       env: TYPO3_VERSION="dev-master as 8.7.0"
   allow_failures:

Classes/Command/InstallCommandController.php

@@ -90,7 +90,7 @@ public function setupCommand(
         $this->cliSetupRequestHandler->setup(!$nonInteractive, $this->request->getArguments());
 
         $this->outputLine();
-        $this->outputLine('Successfully installed TYPO3 CMS!');
+        $this->outputLine('<i>Successfully installed TYPO3 CMS!</i>');
     }
 
     /**

Classes/Command/UpgradeCommandController.php

@@ -0,0 +1,114 @@
+<?php
+namespace Helhum\Typo3Console\Command;
+
+/*
+ * This file is part of the TYPO3 Console project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read
+ * LICENSE file that was distributed with this source code.
+ *
+ */
+
+use Helhum\Typo3Console\Install\Upgrade\UpgradeHandling;
+use Helhum\Typo3Console\Install\Upgrade\UpgradeWizardListRenderer;
+use Helhum\Typo3Console\Install\Upgrade\UpgradeWizardResultRenderer;
+use Helhum\Typo3Console\Mvc\Cli\CommandDispatcher;
+use Helhum\Typo3Console\Mvc\Controller\CommandController;
+
+class UpgradeCommandController extends CommandController
+{
+    /**
+     * @var UpgradeHandling
+     */
+    private $upgradeHandling;
+
+    /**
+     * @var CommandDispatcher
+     */
+    private $commandDispatcher;
+
+    /**
+     * @param UpgradeHandling|null $upgradeHandling
+     * @param CommandDispatcher|null $commandDispatcher
+     */
+    public function __construct(
+        UpgradeHandling $upgradeHandling = null,
+        CommandDispatcher $commandDispatcher = null
+    ) {
+        $this->upgradeHandling = $upgradeHandling ?: new UpgradeHandling();
+        $this->commandDispatcher = $commandDispatcher ?: CommandDispatcher::createFromCommandRun();
+    }
+
+    /**
+     * List upgrade wizards
+     *
+     * @param bool $verbose If set, a more verbose description for each wizard is shown, if not set only the title is shown
+     * @param bool $all If set, all wizards will be listed, even the once marked as ready or done
+     */
+    public function listCommand($verbose = false, $all = false)
+    {
+        $wizards = $this->upgradeHandling->executeInSubProcess('listWizards');
+
+        $listRenderer = new UpgradeWizardListRenderer();
+        $this->outputLine('<comment>Wizards scheduled for execution:</comment>');
+        $listRenderer->render($wizards['scheduled'], $this->output, $verbose);
+
+        if ($all) {
+            $this->outputLine(PHP_EOL . '<comment>Wizards marked as done:</comment>');
+            $listRenderer->render($wizards['done'], $this->output, $verbose);
+        }
+    }
+
+    /**
+     * Execute a single upgrade wizard
+     *
+     * @param string $identifier Identifier of the wizard that should be executed
+     * @param array $arguments Arguments for the wizard prefixed with the identifier, e.g. <code>compatibility7Extension[install]=0</code>
+     * @param bool $force Force execution, even if the wizard has been marked as done
+     */
+    public function wizardCommand($identifier, array $arguments = [], $force = false)
+    {
+        $result = $this->upgradeHandling->executeInSubProcess('executeWizard', [$identifier, $arguments, $force]);
+        (new UpgradeWizardResultRenderer())->render([$identifier => $result], $this->output);
+    }
+
+    /**
+     * Execute all upgrade wizards that are scheduled for execution
+     *
+     * @param array $arguments Arguments for the wizard prefixed with the identifier, e.g. <code>compatibility7Extension[install]=0</code>; multiple arguments separated with comma
+     * @param bool $verbose If set, output of the wizards will be shown, including all SQL Queries that were executed
+     */
+    public function allCommand(array $arguments = [], $verbose = false)
+    {
+        $this->outputLine('<i>Initiating TYPO3 upgrade</i>' . PHP_EOL);
+
+        $results = $this->upgradeHandling->executeAll($arguments, $this->output);
+
+        $this->outputLine(PHP_EOL . PHP_EOL . '<i>Successfully upgraded TYPO3 to version %s</i>', [TYPO3_version]);
+
+        if ($verbose) {
+            $this->outputLine();
+            $this->outputLine('<comment>Upgrade report:</comment>');
+            (new UpgradeWizardResultRenderer())->render($results, $this->output);
+        }
+    }
+
+    /**
+     * This is where the hard work happens in a fully bootstrapped TYPO3
+     * It will be called as sub process
+     *
+     * @param string $command
+     * @param string $arguments Serialized arguments
+     * @internal
+     */
+    public function subProcessCommand($command, $arguments)
+    {
+        $arguments = unserialize($arguments);
+        $result = call_user_func_array([$this->upgradeHandling, $command], $arguments);
+        $this->output(serialize($result));
+    }
+}

Classes/Composer/InstallerScript/GeneratePackageStates.php

@@ -71,9 +71,9 @@ public function run(ScriptEvent $event)
             'frameworkExtensions' => (string)getenv('TYPO3_ACTIVE_FRAMEWORK_EXTENSIONS'),
         ];
         if (getenv('TYPO3_ACTIVATE_DEFAULT_FRAMEWORK_EXTENSIONS')) {
-            $commandOptions['activateDefault'] = null;
+            $commandOptions['activateDefault'] = true;
         }
-        if ($event->isDevMode()) {
+        if ($event->isDevMode() && getenv('TYPO3_EXCLUDED_EXTENSIONS')) {
             $commandOptions['excludedExtensions'] = (string)getenv('TYPO3_EXCLUDED_EXTENSIONS');
         }
         $output = $commandDispatcher->executeCommand('install:generatepackagestates', $commandOptions);

Classes/Install/InstallStepActionExecutor.php

@@ -37,6 +37,7 @@ class InstallStepActionExecutor
 
     /**
      * @param ObjectManager $objectManager
+     * @param SilentConfigurationUpgrade $silentConfigurationUpgrade
      */
     public function __construct(ObjectManager $objectManager, SilentConfigurationUpgrade $silentConfigurationUpgrade)
     {

Classes/Install/Upgrade/UpgradeHandling.php

@@ -0,0 +1,233 @@
+<?php
+namespace Helhum\Typo3Console\Install\Upgrade;
+
+/*
+ * This file is part of the TYPO3 Console project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read
+ * LICENSE file that was distributed with this source code.
+ *
+ */
+
+use Helhum\Typo3Console\Core\ConsoleBootstrap;
+use Helhum\Typo3Console\Mvc\Cli\CommandDispatcher;
+use Helhum\Typo3Console\Mvc\Cli\ConsoleOutput;
+use Helhum\Typo3Console\Service\Configuration\ConfigurationService;
+use TYPO3\CMS\Install\Updates\DatabaseCharsetUpdate;
+
+/**
+ * Executes a single upgrade wizard
+ * Holds the information on possible user interactions
+ */
+class UpgradeHandling
+{
+    /**
+     * @var UpgradeWizardExecutor
+     */
+    private $executor;
+
+    /**
+     * @var SilentConfigurationUpgrade
+     */
+    private $silentConfigurationUpgrade;
+
+    /**
+     * @var CommandDispatcher
+     */
+    private $commandDispatcher;
+
+    /**
+     * @var UpgradeWizardList
+     */
+    private $upgradeWizardList;
+
+    /**
+     * @var ConfigurationService|null
+     */
+    private $configurationService;
+
+    /**
+     * Flag for same process
+     *
+     * @var bool
+     */
+    private $initialUpgradeDone = false;
+
+    /**
+     * Wizards that have a user interaction with resulting argument
+     *
+     * @var array
+     */
+    private static $wizardsWithArguments = [
+        'DbalAndAdodbExtractionUpdate' => [['name' => 'install', 'type' => 'bool', 'default' => '0']],
+        'compatibility7Extension' => [['name' => 'install', 'type' => 'bool', 'default' => '0']],
+        'rtehtmlareaExtension' => [['name' => 'install', 'type' => 'bool', 'default' => '0']],
+    ];
+    /**
+     * @var UpgradeWizardFactory|null
+     */
+    private $factory;
+
+    /**
+     * @param UpgradeWizardFactory|null $factory
+     * @param UpgradeWizardExecutor $executor
+     * @param UpgradeWizardList|null $upgradeWizardList
+     * @param SilentConfigurationUpgrade|null $silentConfigurationUpgrade
+     * @param CommandDispatcher|null $commandDispatcher
+     * @param ConfigurationService|null $configurationService
+     */
+    public function __construct(
+        UpgradeWizardFactory $factory = null,
+        UpgradeWizardExecutor $executor = null,
+        UpgradeWizardList $upgradeWizardList = null,
+        SilentConfigurationUpgrade $silentConfigurationUpgrade = null,
+        CommandDispatcher $commandDispatcher = null,
+        ConfigurationService $configurationService = null
+    ) {
+        $this->factory = new UpgradeWizardFactory();
+        $this->executor = $executor ?: new UpgradeWizardExecutor($this->factory);
+        $this->upgradeWizardList = $upgradeWizardList ?: new UpgradeWizardList();
+        $this->silentConfigurationUpgrade = $silentConfigurationUpgrade ?: new SilentConfigurationUpgrade();
+        $this->commandDispatcher = $commandDispatcher ?: CommandDispatcher::createFromCommandRun();
+        $this->configurationService = $configurationService ?: new ConfigurationService();
+    }
+
+    /**
+     * @param string $identifier
+     * @param array $rawArguments
+     * @param bool $force
+     * @return UpgradeWizardResult
+     */
+    public function executeWizard($identifier, array $rawArguments = [], $force = false)
+    {
+        return $this->executor->executeWizard($identifier, $rawArguments, $force);
+    }
+
+    /**
+     * @param array $arguments
+     * @param ConsoleOutput|null $consoleOutput
+     * @return array
+     */
+    public function executeAll(array $arguments, ConsoleOutput $consoleOutput = null)
+    {
+        if ($consoleOutput) {
+            $consoleOutput->progressStart(rand(6, 9));
+            $consoleOutput->progressAdvance();
+        }
+
+        $wizards = $this->executeInSubProcess('listWizards');
+
+        if ($consoleOutput) {
+            $consoleOutput->progressStart(count($wizards['scheduled']) + 2);
+        }
+
+        $results = [];
+        if (!empty($wizards['scheduled'])) {
+            foreach ($wizards['scheduled'] as $identifier => $_) {
+                if ($consoleOutput) {
+                    $consoleOutput->progressAdvance();
+                }
+                $shortIdentifier = str_replace('TYPO3\\CMS\\Install\\Updates\\', '', $identifier);
+                if ($consoleOutput && isset(self::$wizardsWithArguments[$shortIdentifier])
+                ) {
+                    foreach (self::$wizardsWithArguments[$shortIdentifier] as $argumentDefinition) {
+                        $argumentName = $argumentDefinition['name'];
+                        $argumentDefault = $argumentDefinition['default'];
+                        if ($this->wizardHasArgument($shortIdentifier, $argumentName, $arguments)) {
+                            continue;
+                        }
+                        // In composer mode, skip all install extension wizards!
+                        if (ConsoleBootstrap::usesComposerClassLoading()) {
+                            $arguments[] = sprintf('%s[%s]=%s', $shortIdentifier, $argumentName, $argumentDefault);
+                        } elseif ($argumentDefinition['type'] === 'bool') {
+                            $wizard = $this->factory->create($shortIdentifier);
+                            $consoleOutput->outputLine(PHP_EOL . PHP_EOL . '<info>' . $wizard->getTitle() . '</info>' . PHP_EOL);
+                            $consoleOutput->outputLine(implode(PHP_EOL, array_filter(array_map('trim', explode(chr(10), html_entity_decode(strip_tags($wizard->getUserInput(''))))))));
+                            $consoleOutput->outputLine();
+                            $arguments[] = sprintf(
+                                '%s[%s]=%s',
+                                    $shortIdentifier,
+                                    $argumentName,
+                                    (string)(int)$consoleOutput->askConfirmation('<comment>Install (y/N)</comment> ', $argumentDefault)
+                            );
+                        }
+                    }
+                }
+                $results[$identifier] = $this->executeInSubProcess('executeWizard', [$identifier, $arguments]);
+            }
+        }
+
+        if ($consoleOutput) {
+            $consoleOutput->progressAdvance();
+        }
+
+        $this->commandDispatcher->executeCommand('database:updateschema');
+
+        if ($consoleOutput) {
+            $consoleOutput->progressFinish();
+        }
+
+        return $results;
+    }
+
+    /**
+     * @param string $identifier
+     * @param string $argumentName
+     * @param array $arguments
+     * @return bool
+     */
+    private function wizardHasArgument($identifier, $argumentName, array $arguments)
+    {
+        if (isset(self::$wizardsWithArguments[$identifier])) {
+            foreach ($arguments as $argument) {
+                if (strpos($argument, sprintf('%s[%s]', $identifier, $argumentName)) !== false) {
+                    return true;
+                }
+                if (strpos($argument, '[') === false && strpos($argument, $argumentName) !== false) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    /**
+     * @return array
+     */
+    public function listWizards()
+    {
+        return [
+            'scheduled' => $this->upgradeWizardList->listWizards(),
+            'done' => $this->upgradeWizardList->listWizards(true),
+        ];
+    }
+
+    /**
+     * Execute the command in a sub process,
+     * but execute some automated migration steps beforehand
+     *
+     * @param string $command
+     * @param array $arguments
+     * @return mixed
+     */
+    public function executeInSubProcess($command, array $arguments = [])
+    {
+        $this->ensureUpgradeIsPossible();
+        return @unserialize($this->commandDispatcher->executeCommand('upgrade:subprocess', ['command' => $command, 'arguments' => serialize($arguments)]));
+    }
+
+    private function ensureUpgradeIsPossible()
+    {
+        if (!$this->initialUpgradeDone && !$this->configurationService->hasActive('EXTCONF/helhum-typo3-console/initialUpgradeDone')) {
+            $this->initialUpgradeDone = true;
+            $this->configurationService->setLocal('EXTCONF/helhum-typo3-console/initialUpgradeDone', true);
+            $this->silentConfigurationUpgrade->executeSilentConfigurationUpgradesIfNeeded();
+            $this->commandDispatcher->executeCommand('upgrade:executewizard', ['identifier' => DatabaseCharsetUpdate::class]);
+            $this->commandDispatcher->executeCommand('database:updateschema');
+        }
+    }
+}