Move the installer into the core

UPGRADE.md

@@ -390,3 +390,10 @@ The public folder is now called `public` by default. It can be renamed in the `c
 The `Contao\CoreBundle\Image\Studio\Figure::getLinkAttributes()` method will now return an
 `Contao\CoreBundle\String\HtmlAttributes` object instead of an array. Use `iterator_to_array()` to transform it
 back to an array representation. If you are just using array access, nothing needs to be changed.
+
+### Install Tool
+
+The ability to execute migrations has been removed from the Install Tool, use the `contao:migrate` command or the
+Contao Manager instead.
+
+The `sqlCompileCommands` hook has been removed, use the Doctrine DBAL `postGenerateSchema` event instead.

core-bundle/src/Command/MigrateCommand.php

@@ -15,9 +15,10 @@
 use Contao\CoreBundle\Doctrine\Backup\BackupManager;
 use Contao\CoreBundle\Doctrine\Schema\MysqlInnodbRowSizeCalculator;
 use Contao\CoreBundle\Doctrine\Schema\SchemaProvider;
+use Contao\CoreBundle\Migration\CommandCompiler;
 use Contao\CoreBundle\Migration\MigrationCollection;
 use Contao\CoreBundle\Migration\MigrationResult;
-use Contao\InstallationBundle\Database\Installer;
+use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Schema\Table;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Exception\InvalidOptionException;
@@ -34,11 +35,12 @@ class MigrateCommand extends Command
     private SymfonyStyle|null $io = null;
 
     public function __construct(
+        private CommandCompiler $commandCompiler,
+        private Connection $connection,
         private MigrationCollection $migrations,
         private BackupManager $backupManager,
         private SchemaProvider $schemaProvider,
         private MysqlInnodbRowSizeCalculator $rowSizeCalculator,
-        private Installer|null $installer = null,
     ) {
         parent::__construct();
     }
@@ -253,12 +255,6 @@ private function executeMigrations(bool &$dryRun, bool $asJson, string $specifie
 
     private function executeSchemaDiff(bool $dryRun, bool $asJson, bool $withDeletesOption, string $specifiedHash = null): bool
     {
-        if (null === $this->installer) {
-            $this->io->error('Service "contao_installation.database.installer" not found. The installation bundle needs to be installed in order to execute schema diff migrations.');
-
-            return false;
-        }
-
         if ($schemaWarnings = $this->compileSchemaWarnings()) {
             $this->io->warning(implode("\n\n", $schemaWarnings));
 
@@ -267,25 +263,20 @@ private function executeSchemaDiff(bool $dryRun, bool $asJson, bool $withDeletes
             }
         }
 
-        $commandsByHash = [];
+        $lastCommands = [];
 
         while (true) {
-            $this->installer->compileCommands();
+            $commands = $this->commandCompiler->compileCommands();
 
-            $commands = $this->installer->getCommands(false);
+            $hasNewCommands = \count(array_diff($commands, $lastCommands)) > 0;
+            $lastCommands = $commands;
 
-            $hasNewCommands = \count(array_filter(
-                array_keys($commands),
-                static fn ($hash) => !isset($commandsByHash[$hash])
-            ));
-
-            $commandsByHash = $commands;
-            $actualHash = hash('sha256', json_encode($commands));
+            $commandsHash = hash('sha256', json_encode($commands));
 
             if ($asJson) {
                 $this->writeNdjson('schema-pending', [
-                    'commands' => array_values($commandsByHash),
-                    'hash' => $actualHash,
+                    'commands' => $commands,
+                    'hash' => $commandsHash,
                 ]);
             }
 
@@ -294,16 +285,16 @@ private function executeSchemaDiff(bool $dryRun, bool $asJson, bool $withDeletes
             }
 
             if (!$asJson) {
-                $this->io->section("Pending database migrations ($actualHash)");
-                $this->io->listing($commandsByHash);
+                $this->io->section("Pending database migrations ($commandsHash)");
+                $this->io->listing($commands);
             }
 
             if ($dryRun) {
                 return true;
             }
 
-            if (null !== $specifiedHash && $specifiedHash !== $actualHash) {
-                throw new InvalidOptionException(sprintf('Specified hash "%s" does not match the actual hash "%s"', $specifiedHash, $actualHash));
+            if (null !== $specifiedHash && $specifiedHash !== $commandsHash) {
+                throw new InvalidOptionException(sprintf('Specified hash "%s" does not match the actual hash "%s"', $specifiedHash, $commandsHash));
             }
 
             $options = $withDeletesOption
@@ -325,31 +316,35 @@ private function executeSchemaDiff(bool $dryRun, bool $asJson, bool $withDeletes
             }
 
             $count = 0;
-            $commandHashes = $this->getCommandHashes($commands, 'yes, with deletes' === $answer);
+
+            // If deletes should not be processed, recompile the commands without drop statements
+            if ('yes, with deletes' !== $answer) {
+                $commands = $this->commandCompiler->compileCommands(true);
+            }
 
             do {
                 $commandExecuted = false;
                 $exceptions = [];
 
-                foreach ($commandHashes as $key => $hash) {
+                foreach ($commands as $key => $command) {
                     if ($asJson) {
                         $this->writeNdjson('schema-execute', [
-                            'command' => $commandsByHash[$hash],
+                            'command' => $command,
                         ]);
                     } else {
-                        $this->io->write(' * '.$commandsByHash[$hash]);
+                        $this->io->write(' * '.$command);
                     }
 
                     try {
-                        $this->installer->execCommand($hash);
+                        $this->connection->executeQuery($command);
 
                         ++$count;
                         $commandExecuted = true;
-                        unset($commandHashes[$key]);
+                        unset($commands[$key]);
 
                         if ($asJson) {
                             $this->writeNdjson('schema-result', [
-                                'command' => $commandsByHash[$hash],
+                                'command' => $command,
                                 'isSuccessful' => true,
                             ]);
                         } else {
@@ -360,7 +355,7 @@ private function executeSchemaDiff(bool $dryRun, bool $asJson, bool $withDeletes
 
                         if ($asJson) {
                             $this->writeNdjson('schema-result', [
-                                'command' => $commandsByHash[$hash],
+                                'command' => $command,
                                 'isSuccessful' => false,
                                 'message' => $e->getMessage(),
                             ]);
@@ -394,22 +389,6 @@ private function executeSchemaDiff(bool $dryRun, bool $asJson, bool $withDeletes
         return true;
     }
 
-    private function getCommandHashes(array $commands, bool $withDrops): array
-    {
-        if (!$withDrops) {
-            foreach ($commands as $hash => $command) {
-                if (
-                    preg_match('/^ALTER TABLE [^ ]+ DROP /', (string) $command)
-                    || (str_starts_with($command, 'DROP ') && !str_starts_with($command, 'DROP INDEX'))
-                ) {
-                    unset($commands[$hash]);
-                }
-            }
-        }
-
-        return array_keys($commands);
-    }
-
     private function writeNdjson(string $type, array $data): void
     {
         $this->io->writeln(

core-bundle/src/Migration/CommandCompiler.php

@@ -0,0 +1,205 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Contao.
+ *
+ * (c) Leo Feyer
+ *
+ * @license LGPL-3.0-or-later
+ */
+
+namespace Contao\CoreBundle\Migration;
+
+use Contao\CoreBundle\Doctrine\Schema\SchemaProvider;
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\Schema\Column;
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Schema\Table;
+
+class CommandCompiler
+{
+    /**
+     * @internal Do not inherit from this class; decorate the "contao.migration.command_compiler" service instead
+     */
+    public function __construct(private readonly Connection $connection, private readonly SchemaProvider $schemaProvider)
+    {
+    }
+
+    /**
+     * @return list<string>
+     */
+    public function compileCommands(bool $skipDropStatements = false): array
+    {
+        $schemaManager = $this->connection->createSchemaManager();
+        $fromSchema = $schemaManager->createSchema();
+        $toSchema = $this->schemaProvider->createSchema();
+
+        // If tables or columns should be preserved, we copy the missing
+        // definitions over to the $toSchema, so that no DROP commands
+        // will be issued in the diff.
+        if ($skipDropStatements) {
+            foreach ($fromSchema->getTables() as $table) {
+                if (!$toSchema->hasTable($table->getName())) {
+                    $this->copyTableDefinition($toSchema, $table);
+
+                    continue;
+                }
+
+                $toSchemaTable = $toSchema->getTable($table->getName());
+
+                foreach ($table->getColumns() as $column) {
+                    if (!$toSchemaTable->hasColumn($column->getName())) {
+                        $this->copyColumnDefinition($toSchemaTable, $column);
+                    }
+                }
+            }
+        }
+
+        // Get a list of SQL statements from the schema diff
+        $diffCommands = $schemaManager
+            ->createComparator()
+            ->compareSchemas($fromSchema, $toSchema)
+            ->toSql($this->connection->getDatabasePlatform())
+        ;
+
+        // Get a list of SQL statements that adjust the engine and collation options
+        $engineAndCollationCommands = $this->compileEngineAndCollationCommands($fromSchema, $toSchema);
+
+        return array_unique([...$diffCommands, ...$engineAndCollationCommands]);
+    }
+
+    private function copyTableDefinition(Schema $targetSchema, Table $table): void
+    {
+        (new \ReflectionClass(Schema::class))
+            ->getMethod('_addTable')
+            ->invoke($targetSchema, $table)
+        ;
+    }
+
+    private function copyColumnDefinition(Table $targetTable, Column $column): void
+    {
+        (new \ReflectionClass(Table::class))
+            ->getMethod('_addColumn')
+            ->invoke($targetTable, $column)
+        ;
+    }
+
+    /**
+     * Checks engine and collation and adds the ALTER TABLE queries.
+     *
+     * @return list<string>
+     */
+    private function compileEngineAndCollationCommands(Schema $fromSchema, Schema $toSchema): array
+    {
+        $tables = $toSchema->getTables();
+        $dynamic = $this->hasDynamicRowFormat();
+
+        $commands = [];
+
+        foreach ($tables as $table) {
+            $tableName = $table->getName();
+            $deleteIndexes = false;
+
+            if (!str_starts_with($tableName, 'tl_')) {
+                continue;
+            }
+
+            $tableOptions = $this->connection->fetchAssociative(
+                'SHOW TABLE STATUS WHERE Name = ? AND Engine IS NOT NULL AND Create_options IS NOT NULL AND Collation IS NOT NULL',
+                [$tableName]
+            );
+
+            if (false === $tableOptions) {
+                continue;
+            }
+
+            $engine = $table->hasOption('engine') ? $table->getOption('engine') : '';
+            $innodb = 'innodb' === strtolower($engine);
+
+            if (strtolower($tableOptions['Engine']) !== strtolower($engine)) {
+                if ($innodb && $dynamic) {
+                    $command = 'ALTER TABLE '.$tableName.' ENGINE = '.$engine.' ROW_FORMAT = DYNAMIC';
+
+                    if (false !== stripos($tableOptions['Create_options'], 'key_block_size=')) {
+                        $command .= ' KEY_BLOCK_SIZE = 0';
+                    }
+                } else {
+                    $command = 'ALTER TABLE '.$tableName.' ENGINE = '.$engine;
+                }
+
+                $deleteIndexes = true;
+                $commands[] = $command;
+            } elseif ($innodb && $dynamic) {
+                if (false === stripos($tableOptions['Create_options'], 'row_format=dynamic')) {
+                    $command = 'ALTER TABLE '.$tableName.' ENGINE = '.$engine.' ROW_FORMAT = DYNAMIC';
+
+                    if (false !== stripos($tableOptions['Create_options'], 'key_block_size=')) {
+                        $command .= ' KEY_BLOCK_SIZE = 0';
+                    }
+
+                    $commands[] = $command;
+                }
+            }
+
+            $collate = '';
+            $charset = $table->hasOption('charset') ? $table->getOption('charset') : '';
+
+            if ($table->hasOption('collation')) {
+                $collate = $table->getOption('collation');
+            } elseif ($table->hasOption('collate')) {
+                $collate = $table->getOption('collate');
+            }
+
+            if ($tableOptions['Collation'] !== $collate && '' !== $charset) {
+                $command = 'ALTER TABLE '.$tableName.' CONVERT TO CHARACTER SET '.$charset.' COLLATE '.$collate;
+                $deleteIndexes = true;
+                $commands[] = $command;
+            }
+
+            // Delete the indexes if the engine changes in case the existing
+            // indexes are too long. The migration then needs to be run multiple
+            // times to re-create the indexes with the correct length.
+            if ($deleteIndexes) {
+                if (!$fromSchema->hasTable($tableName)) {
+                    continue;
+                }
+
+                $platform = $this->connection->getDatabasePlatform();
+
+                foreach ($fromSchema->getTable($tableName)->getIndexes() as $index) {
+                    $indexName = $index->getName();
+
+                    if ('primary' === strtolower($indexName)) {
+                        continue;
+                    }
+
+                    $commands[] = $platform->getDropIndexSQL($indexName, $tableName);
+                }
+            }
+        }
+
+        return $commands;
+    }
+
+    private function hasDynamicRowFormat(): bool
+    {
+        $filePerTable = $this->connection->fetchAssociative("SHOW VARIABLES LIKE 'innodb_file_per_table'");
+
+        // Dynamic rows require innodb_file_per_table to be enabled
+        if (!\in_array(strtolower((string) $filePerTable['Value']), ['1', 'on'], true)) {
+            return false;
+        }
+
+        $fileFormat = $this->connection->fetchAssociative("SHOW VARIABLES LIKE 'innodb_file_format'");
+
+        // MySQL 8 and MariaDB 10.3 no longer have the "innodb_file_format" setting
+        if (false === $fileFormat || '' === $fileFormat['Value']) {
+            return true;
+        }
+
+        // Dynamic rows require the Barracuda file format in MySQL <8 and MariaDB <10.3
+        return 'barracuda' === strtolower((string) $fileFormat['Value']);
+    }
+}

core-bundle/src/Resources/config/commands.yaml

@@ -86,11 +86,12 @@ services:
     contao.command.migrate:
         class: Contao\CoreBundle\Command\MigrateCommand
         arguments:
+            - '@contao.migration.command_compiler'
+            - '@database_connection'
             - '@contao.migration.collection'
             - '@contao.doctrine.backup_manager'
             - '@contao.doctrine.schema_provider'
             - '@contao.doctrine.schema.mysql_innodb_row_size_calculator'
-            - '@?contao_installation.database.installer'
 
     contao.command.resize_images:
         class: Contao\CoreBundle\Command\ResizeImagesCommand