MySQL: convert/use row format dynamic (Fixes #362)

CHANGELOG.md

@@ -6,6 +6,9 @@
   on the server are discovered and added, whereas addressbooks that have been removed from the server are also removed
   from roundcube. For manually-added addressbooks, this will require changes to the rcmcarddav data model, which is
   planned for version 5.
+- MySQL: Convert potentially used row format COMPACT (was default up to MySQL 5.7.8, Maria DB 10.2.1) to DYANMIC in
+  migration 12, which would otherwise fail (Fixes #362). It requires some other settings that have to be configured in
+  the MySQL server configuration additionally, all of which are also defaults since MySQL 5.7.7 / Maria DB 10.2.2.
 
 ## Version 4.2.2 (to 4.2.1)
 

README.md

@@ -28,6 +28,11 @@ A (hopefully growing) documentation for various topics is found in the [doc](doc
 - Database migration happens automatically.
 - If you want more verbose than default logging, this must now be configured in `config.inc.php`. See the distributed
   file `config.inc.php.dist` for examples.
+- For MySQL / Maria DB: If your database was created with MySQL 5.7.8 / MariaDB 10.2.1 or earlier, it likely uses the
+  `COMPACT` row format. This makes a DB migration fail, because the index size is exceeded. Migration 12 since
+  rcmcarddav 4.3.0 converts the row format to the current default `DYNAMIC`, but some additional settings are required
+  in the MySQL / Maria DB configuration for increase the index key limit to 3072 bytes. See [INSTALL.md](doc/INSTALL.md)
+  for these settings.
 
 ### Upgrading from 2.0.x
 

dbmigrations/0000-dbinit/mysql.sql

@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_addressbooks (
 
 	PRIMARY KEY(id),
 	FOREIGN KEY (user_id) REFERENCES TABLE_PREFIXusers(user_id) ON DELETE CASCADE ON UPDATE CASCADE
-) CHARACTER SET utf8 COLLATE utf8_unicode_ci /*!40000 ENGINE=INNODB */;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_contacts (
 	id INT UNSIGNED NOT NULL AUTO_INCREMENT,
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_contacts (
 	UNIQUE INDEX(uri,abook_id),
 	UNIQUE INDEX(cuid,abook_id),
 	FOREIGN KEY (abook_id) REFERENCES TABLE_PREFIXcarddav_addressbooks(id) ON DELETE CASCADE ON UPDATE CASCADE
-) CHARACTER SET utf8 COLLATE utf8_unicode_ci /*!40000 ENGINE=INNODB */;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_xsubtypes (
 	id INT UNSIGNED NOT NULL AUTO_INCREMENT,
@@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_xsubtypes (
 	PRIMARY KEY(id),
 	UNIQUE INDEX(typename,subtype,abook_id),
 	FOREIGN KEY (abook_id) REFERENCES TABLE_PREFIXcarddav_addressbooks(id) ON DELETE CASCADE ON UPDATE CASCADE
-) CHARACTER SET utf8 COLLATE utf8_unicode_ci /*!40000 ENGINE=INNODB */;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_groups (
 	id INT UNSIGNED NOT NULL AUTO_INCREMENT,
@@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_groups (
 	UNIQUE(cuid,abook_id),
 
 	FOREIGN KEY (abook_id) REFERENCES TABLE_PREFIXcarddav_addressbooks(id) ON DELETE CASCADE ON UPDATE CASCADE
-) CHARACTER SET utf8 COLLATE utf8_unicode_ci /*!40000 ENGINE=INNODB */;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_group_user (
 	group_id   INT UNSIGNED NOT NULL,
@@ -72,7 +72,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_group_user (
 	PRIMARY KEY(group_id,contact_id),
 	FOREIGN KEY(group_id) REFERENCES TABLE_PREFIXcarddav_groups(id) ON DELETE CASCADE ON UPDATE CASCADE,
 	FOREIGN KEY(contact_id) REFERENCES TABLE_PREFIXcarddav_contacts(id) ON DELETE CASCADE ON UPDATE CASCADE
-) CHARACTER SET utf8 COLLATE utf8_unicode_ci /*!40000 ENGINE=INNODB */;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;
 
 CREATE TABLE TABLE_PREFIXcarddav_migrations (
 	`ID` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY ,
@@ -81,4 +81,4 @@ CREATE TABLE TABLE_PREFIXcarddav_migrations (
 	UNIQUE (
 		`filename`
 	)
-) ENGINE = InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;

dbmigrations/0012-fixmysqlconstraints/README.md

@@ -1,3 +1,6 @@
 In migration 0005, the NOT NULL constraints are missing from most of the altered column
 definitions. This migrations re-adds them. Furthermore, it unifies the indexes to that
 the schema after migrations equals the INIT schema.
+
+2021-11-01: Because of issue 362, I added statements to the MySQL variant of this migration that convert the row format
+to dynamic. This is a prerequisite for index keys up to 3072 bytes.

dbmigrations/0012-fixmysqlconstraints/mysql.sql

@@ -1,3 +1,22 @@
+/*
+ * Convert the ROW_FORMAT to DYNAMIC. This, in combination with some other settings, which
+ * are nowadays default values (from MySQL 5.7.9, Maria DB 10.2.2), allows for index prefixes
+ * up to 3072 bytes. The compact format only allows up to 767 bytes.
+ *
+ * If the 767 limit applies, the UNIQUE indexes re-created below containing URI fields
+ * in their key could theoretically exceed the limit, and thus MySQL will fail with an
+ * error.
+ *
+ * See https://github.com/mstilkerich/rcmcarddav/issues/362
+ */
+ALTER TABLE TABLE_PREFIXcarddav_addressbooks ROW_FORMAT=DYNAMIC;
+ALTER TABLE TABLE_PREFIXcarddav_contacts ROW_FORMAT=DYNAMIC;
+ALTER TABLE TABLE_PREFIXcarddav_xsubtypes ROW_FORMAT=DYNAMIC;
+ALTER TABLE TABLE_PREFIXcarddav_groups ROW_FORMAT=DYNAMIC;
+ALTER TABLE TABLE_PREFIXcarddav_group_user ROW_FORMAT=DYNAMIC;
+ALTER TABLE TABLE_PREFIXcarddav_migrations ROW_FORMAT=DYNAMIC;
+
+
 ALTER TABLE TABLE_PREFIXcarddav_addressbooks MODIFY `name` VARCHAR(64) NOT NULL;
 ALTER TABLE TABLE_PREFIXcarddav_addressbooks MODIFY `username` VARCHAR(255) NOT NULL;
 ALTER TABLE TABLE_PREFIXcarddav_addressbooks MODIFY `password` TEXT NOT NULL;

dbmigrations/INIT-currentschema/mysql.sql

@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_addressbooks (
 	PRIMARY KEY(id),
 	KEY `user_id` (`user_id`) USING BTREE,
 	FOREIGN KEY (user_id) REFERENCES TABLE_PREFIXusers(user_id) ON DELETE CASCADE ON UPDATE CASCADE
-) ENGINE = InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_contacts (
 	id INT UNSIGNED NOT NULL AUTO_INCREMENT,
@@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_contacts (
 	UNIQUE INDEX(uri,abook_id),
 	UNIQUE INDEX(cuid,abook_id),
 	FOREIGN KEY (abook_id) REFERENCES TABLE_PREFIXcarddav_addressbooks(id) ON DELETE CASCADE ON UPDATE CASCADE
-) ENGINE = InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_xsubtypes (
 	id INT UNSIGNED NOT NULL AUTO_INCREMENT,
@@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_xsubtypes (
 	PRIMARY KEY(id),
 	UNIQUE INDEX(typename,subtype,abook_id),
 	FOREIGN KEY (abook_id) REFERENCES TABLE_PREFIXcarddav_addressbooks(id) ON DELETE CASCADE ON UPDATE CASCADE
-) ENGINE = InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_groups (
 	id INT UNSIGNED NOT NULL AUTO_INCREMENT,
@@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_groups (
 	UNIQUE INDEX(cuid,abook_id),
 
 	FOREIGN KEY (abook_id) REFERENCES TABLE_PREFIXcarddav_addressbooks(id) ON DELETE CASCADE ON UPDATE CASCADE
-) ENGINE = InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_group_user (
 	group_id   INT UNSIGNED NOT NULL,
@@ -73,11 +73,11 @@ CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_group_user (
 	PRIMARY KEY(group_id,contact_id),
 	FOREIGN KEY(group_id) REFERENCES TABLE_PREFIXcarddav_groups(id) ON DELETE CASCADE ON UPDATE CASCADE,
 	FOREIGN KEY(contact_id) REFERENCES TABLE_PREFIXcarddav_contacts(id) ON DELETE CASCADE ON UPDATE CASCADE
-) ENGINE = InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
 
 CREATE TABLE IF NOT EXISTS TABLE_PREFIXcarddav_migrations (
 	`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
 	`filename` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
 	`processed_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
 	UNIQUE INDEX(`filename`)
-) ENGINE = InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;
+) ROW_FORMAT=DYNAMIC ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;

doc/INSTALL.md

@@ -10,6 +10,16 @@ There is two ways to install the plugin.
 
 After installation, you may optionally [configure](#configuration) the plugin.
 
+## Prerequisites
+
+- When using MySQL 5.7.7 / Maria DB 10.2.1 or older, the following configuration settings are needed in the database
+  server. Note that these are also required by roundcube, and are the default settings if you use newer versions than
+  those listed above.
+  - `innodb_large_prefix=1`
+  - `innodb_file_per_table=1`
+  - `innodb_file_format=Barracuda`
+  - `innodb_default_row_format=dynamic`
+
 ## Installation using composer
 
 The recommended and supported method of installation is by using composer.
@@ -45,7 +55,8 @@ inside that tarball for the appropriate instructions.
 - Log out of Roundcube!
   This is important because RCMCardDAV runs its database initialisation / update procedure only when a user logs in!
 - Download the release tarball from [here](https://github.com/mstilkerich/rcmcarddav/releases)
-  - Note: The correct tarball is named `carddav-vX.Y.Z.tar.gz`. Do not use the "Source code" tar.gz or zip files, these are only exports of the repository. Unfortunately, github creates these automatically for each release.
+  - Note: The correct tarball is named `carddav-vX.Y.Z.tar.gz`. Do not use the "Source code" tar.gz or zip files, these
+    are only exports of the repository. Unfortunately, github creates these automatically for each release.
 - Extract the tarball to the roundcube/plugins directory (assuming roundcube is installed at `/var/lib/roundcube`)
   `cd /var/lib/roundcube/plugins && tar xvzf /tmp/carddav-v4.1.0.tar.gz`
 - [Configure](#configuration) the plugin if needed.