Add a warning for too large database row sizes (see #4179)

Description
-----------

To improve the situation for #4159 I created a small tool that is able to measure and calculate row sizes.

Not sure if this should ever get merged, but I needed a way to write the (somewhat confusing) rules for row sizes down and be able to validate them.

`getMysqlColumnSizeBits()` and `getInnodbColumnSizeBits()` calculates the amout of bits the column contributes to the row size.

`measureMysqlColumnSizeBits()` and `measureInnodbColumnSizeBits()` return the exact same information, but evaluate the amout of bits needed by trail and error against the database. (slow, only used for testing)

![Bildschirmfoto 2022-02-19 um 23 46 42](https://user-images.githubusercontent.com/367169/154822307-81215715-14db-4156-a683-367eb2ef0ceb.png)

Commits
-------

f672abd3 Add a warning for too large database row sizes
92cba100 Fix column charset detection
d4d4858c Reduce row size of tl_content
7f0d8ce4 Use customSchemaOptions
8db72fa4 Reduce table row size for tl_content and tl_module
6b2d8f47 Merge remote-tracking branch 'origin/4.13' into fix/row-size-too-large
16b60a79 Increase image size length
467c8d07 Increase image size length
43dfbb33 Support custom collations in the DcaSchemaProvider

Co-authored-by: Leo Feyer <github@contao.org>

src/Command/MigrateCommand.php

@@ -13,10 +13,13 @@
 namespace Contao\CoreBundle\Command;
 
 use Contao\CoreBundle\Doctrine\Backup\BackupManager;
+use Contao\CoreBundle\Doctrine\Schema\MysqlInnodbRowSizeCalculator;
+use Contao\CoreBundle\Doctrine\Schema\SchemaProvider;
 use Contao\CoreBundle\Framework\ContaoFramework;
 use Contao\CoreBundle\Migration\MigrationCollection;
 use Contao\CoreBundle\Migration\MigrationResult;
 use Contao\InstallationBundle\Database\Installer;
+use Doctrine\DBAL\Schema\Table;
 use Symfony\Component\Config\Exception\FileLocatorFileNotFoundException;
 use Symfony\Component\Config\FileLocator;
 use Symfony\Component\Console\Command\Command;
@@ -38,16 +41,20 @@ class MigrateCommand extends Command
     private string $projectDir;
     private ContaoFramework $framework;
     private BackupManager $backupManager;
+    private SchemaProvider $schemaProvider;
+    private MysqlInnodbRowSizeCalculator $rowSizeCalculator;
     private ?Installer $installer;
     private ?SymfonyStyle $io = null;
 
-    public function __construct(MigrationCollection $migrations, FileLocator $fileLocator, string $projectDir, ContaoFramework $framework, BackupManager $backupManager, Installer $installer = null)
+    public function __construct(MigrationCollection $migrations, FileLocator $fileLocator, string $projectDir, ContaoFramework $framework, BackupManager $backupManager, SchemaProvider $schemaProvider, MysqlInnodbRowSizeCalculator $rowSizeCalculator, Installer $installer = null)
     {
         $this->migrations = $migrations;
         $this->fileLocator = $fileLocator;
         $this->projectDir = $projectDir;
         $this->framework = $framework;
         $this->backupManager = $backupManager;
+        $this->schemaProvider = $schemaProvider;
+        $this->rowSizeCalculator = $rowSizeCalculator;
         $this->installer = $installer;
 
         parent::__construct();
@@ -301,6 +308,14 @@ private function executeSchemaDiff(bool $dryRun, bool $asJson, bool $withDeletes
             return false;
         }
 
+        if ($schemaWarnings = $this->compileSchemaWarnings()) {
+            $this->io->warning(implode("\n\n", $schemaWarnings));
+
+            if (!$this->io->confirm('Continue regardless of the warnings?')) {
+                return false;
+            }
+        }
+
         $commandsByHash = [];
 
         while (true) {
@@ -447,4 +462,38 @@ private function writeNdjson(string $type, array $data): void
             throw new \JsonException(json_last_error_msg());
         }
     }
+
+    /**
+     * @return array<int,string>
+     */
+    private function compileSchemaWarnings(): array
+    {
+        $warnings = [];
+        $schema = $this->schemaProvider->createSchema();
+
+        foreach ($schema->getTables() as $table) {
+            $warnings = [...$warnings, ...$this->compileTableWarnings($table)];
+        }
+
+        return $warnings;
+    }
+
+    /**
+     * @return array<int,string>
+     */
+    private function compileTableWarnings(Table $table): array
+    {
+        $warnings = [];
+
+        $mysqlSize = $this->rowSizeCalculator->getMysqlRowSize($table);
+        $mysqlLimit = $this->rowSizeCalculator->getMysqlRowSizeLimit();
+        $innodbSize = $this->rowSizeCalculator->getInnodbRowSize($table);
+        $innodbLimit = $this->rowSizeCalculator->getInnodbRowSizeLimit();
+
+        if ($mysqlSize > $mysqlLimit || $innodbSize > $innodbLimit) {
+            $warnings[] = "The row size of table {$table->getName()} is too large:\n - MySQL row size: $mysqlSize of $mysqlLimit bytes\n - InnoDB row size: $innodbSize of $innodbLimit bytes";
+        }
+
+        return $warnings;
+    }
 }

src/Doctrine/Schema/DcaSchemaProvider.php

@@ -87,6 +87,18 @@ public function appendToSchema(Schema $schema): void
                         $options['platformOptions']['collation'] = $this->getBinaryCollation($table);
                     }
 
+                    if (isset($options['customSchemaOptions']['charset'])) {
+                        $options['platformOptions']['charset'] = $options['customSchemaOptions']['charset'];
+                    }
+
+                    if (isset($options['customSchemaOptions']['collation'])) {
+                        if (!isset($options['customSchemaOptions']['charset'])) {
+                            $options['platformOptions']['charset'] = explode('_', $options['customSchemaOptions']['collation'], 2)[0];
+                        }
+
+                        $options['platformOptions']['collation'] = $options['customSchemaOptions']['collation'];
+                    }
+
                     $table->addColumn($config['name'], $config['type'], $options);
                 }
             }
@@ -124,6 +136,7 @@ private function parseColumnSql(Table $table, string $columnName, string $sql):
         $scale = null;
         $precision = null;
         $default = null;
+        $charset = null;
         $collation = null;
         $unsigned = false;
         $notnull = false;
@@ -148,6 +161,7 @@ private function parseColumnSql(Table $table, string $columnName, string $sql):
             }
 
             if (preg_match('/collate ([^ ]+)/i', $def, $match)) {
+                $charset = explode('_', $match[1], 2)[0];
                 $collation = $match[1];
             }
 
@@ -178,8 +192,18 @@ private function parseColumnSql(Table $table, string $columnName, string $sql):
             $options['precision'] = $precision;
         }
 
+        $platformOptions = [];
+
+        if (null !== $charset) {
+            $platformOptions['charset'] = $charset;
+        }
+
         if (null !== $collation) {
-            $options['platformOptions'] = ['collation' => $collation];
+            $platformOptions['collation'] = $collation;
+        }
+
+        if (!empty($platformOptions)) {
+            $options['platformOptions'] = $platformOptions;
         }
 
         $table->addColumn($columnName, $type, $options);