From 6c69cd2aea04a503a9e76b9e8e05a89edfa77daf Mon Sep 17 00:00:00 2001 From: nossr50 Date: Tue, 9 Mar 2021 15:10:00 -0800 Subject: [PATCH] Fixed several SQL bugs + SQL db is now fully UTF-8 compatible --- Changelog.txt | 2 + .../nossr50/database/SQLDatabaseManager.java | 101 ++++++++++++++++-- .../datatypes/database/UpgradeType.java | 3 +- .../nossr50/util/upgrade/UpgradeManager.java | 2 +- .../{upgrades.yml => upgrades_overhaul.yml} | 1 + 5 files changed, 99 insertions(+), 10 deletions(-) rename src/main/resources/{upgrades.yml => upgrades_overhaul.yml} (92%) diff --git a/Changelog.txt b/Changelog.txt index a14f8001f9..1189b47c7c 100644 --- a/Changelog.txt +++ b/Changelog.txt @@ -1,5 +1,7 @@ Version 2.1.180 mcMMO now uses UTF-8 compliant encoding for SQL databases (utf8mb4) + Fixed a bug where mcMMO could in some circumstances fail to update SQL schema and mark it as successful + Renamed updates.yml to updates_overhaul.yml to avoid some potential issues when upgrading from classic Version 2.1.179 Fixed a bug for FlatFile databases where some players with changed nicknames would have their levels not loaded upon login (possibly wiping their data) diff --git a/src/main/java/com/gmail/nossr50/database/SQLDatabaseManager.java b/src/main/java/com/gmail/nossr50/database/SQLDatabaseManager.java index 68b20d5dd0..a7ee6eb95a 100644 --- a/src/main/java/com/gmail/nossr50/database/SQLDatabaseManager.java +++ b/src/main/java/com/gmail/nossr50/database/SQLDatabaseManager.java @@ -26,6 +26,9 @@ public final class SQLDatabaseManager implements DatabaseManager { private static final String ALL_QUERY_VERSION = "total"; + public static final String MOBHEALTHBAR_VARCHAR = "VARCHAR(50)"; + public static final String UUID_VARCHAR = "VARCHAR(36)"; + public static final String USER_VARCHAR = "VARCHAR(40)"; private final String tablePrefix = Config.getInstance().getMySQLTablePrefix(); private final Map cachedUserIDs = new HashMap<>(); @@ -38,7 +41,7 @@ public final class SQLDatabaseManager implements DatabaseManager { private final ReentrantLock massUpdateLock = new ReentrantLock(); - private final String ENCODING = "utf8mb4"; //This is compliant with UTF-8 while "utf8" is not, confusing but this is how it is. + private final String CHARSET_SQL = "utf8mb4"; //This is compliant with UTF-8 while "utf8" is not, confusing but this is how it is. protected SQLDatabaseManager() { String connectionString = "jdbc:mysql://" + Config.getInstance().getMySQLServerName() @@ -816,7 +819,7 @@ private void checkStructure() { + "`lastlogin` int(32) unsigned NOT NULL," + "PRIMARY KEY (`id`)," + "INDEX(`user`(20) ASC)," - + "UNIQUE KEY `uuid` (`uuid`)) DEFAULT CHARSET=" + ENCODING + " AUTO_INCREMENT=1;"); + + "UNIQUE KEY `uuid` (`uuid`)) DEFAULT CHARSET=" + CHARSET_SQL + " AUTO_INCREMENT=1;"); tryClose(createStatement); } tryClose(resultSet); @@ -830,7 +833,7 @@ private void checkStructure() { + "`mobhealthbar` varchar(50) NOT NULL DEFAULT '" + Config.getInstance().getMobHealthbarDefault() + "'," + "`scoreboardtips` int(10) NOT NULL DEFAULT '0'," + "PRIMARY KEY (`user_id`)) " - + "DEFAULT CHARSET=" + ENCODING + ";"); + + "DEFAULT CHARSET=" + CHARSET_SQL + ";"); tryClose(createStatement); } tryClose(resultSet); @@ -855,7 +858,7 @@ private void checkStructure() { + "`blast_mining` int(32) unsigned NOT NULL DEFAULT '0'," + "`chimaera_wing` int(32) unsigned NOT NULL DEFAULT '0'," + "PRIMARY KEY (`user_id`)) " - + "DEFAULT CHARSET=" + ENCODING + ";"); + + "DEFAULT CHARSET=" + CHARSET_SQL + ";"); tryClose(createStatement); } tryClose(resultSet); @@ -883,7 +886,7 @@ private void checkStructure() { + "`alchemy` int(10) unsigned NOT NULL DEFAULT "+startingLevel+"," + "`total` int(10) unsigned NOT NULL DEFAULT "+totalLevel+"," + "PRIMARY KEY (`user_id`)) " - + "DEFAULT CHARSET=" + ENCODING + ";"); + + "DEFAULT CHARSET=" + CHARSET_SQL + ";"); tryClose(createStatement); } tryClose(resultSet); @@ -908,7 +911,7 @@ private void checkStructure() { + "`fishing` int(10) unsigned NOT NULL DEFAULT '0'," + "`alchemy` int(10) unsigned NOT NULL DEFAULT '0'," + "PRIMARY KEY (`user_id`)) " - + "DEFAULT CHARSET=" + ENCODING + ";"); + + "DEFAULT CHARSET=" + CHARSET_SQL + ";"); tryClose(createStatement); } tryClose(resultSet); @@ -1031,12 +1034,14 @@ private void checkDatabaseStructure(Connection connection, UpgradeType upgrade) checkUpgradeAddUniqueChimaeraWing(statement); break; + case SQL_CHARSET_UTF8MB4: + updateCharacterSet(statement); + break; + default: break; } - - mcMMO.getUpgradeManager().setUpgradeCompleted(upgrade); } catch (SQLException ex) { printErrors(ex); @@ -1193,6 +1198,7 @@ private void checkNameUniqueness(final Statement statement) { statement.execute("ALTER TABLE `" + tablePrefix + "users` " + "DROP INDEX `user`," + "ADD INDEX `user` (`user`(20) ASC)"); + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.DROP_NAME_UNIQUENESS); } catch (SQLException ex) { ex.printStackTrace(); } finally { @@ -1203,6 +1209,7 @@ private void checkNameUniqueness(final Statement statement) { private void checkUpgradeAddAlchemy(final Statement statement) throws SQLException { try { statement.executeQuery("SELECT `alchemy` FROM `" + tablePrefix + "skills` LIMIT 1"); + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_ALCHEMY); } catch (SQLException ex) { mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Alchemy..."); @@ -1214,6 +1221,7 @@ private void checkUpgradeAddAlchemy(final Statement statement) throws SQLExcepti private void checkUpgradeAddBlastMiningCooldown(final Statement statement) throws SQLException { try { statement.executeQuery("SELECT `blast_mining` FROM `" + tablePrefix + "cooldowns` LIMIT 1"); + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_BLAST_MINING_COOLDOWN); } catch (SQLException ex) { mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Blast Mining..."); @@ -1224,6 +1232,7 @@ private void checkUpgradeAddBlastMiningCooldown(final Statement statement) throw private void checkUpgradeAddUniqueChimaeraWing(final Statement statement) throws SQLException { try { statement.executeQuery("SELECT `chimaera_wing` FROM `" + tablePrefix + "cooldowns` LIMIT 1"); + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_UNIQUE_PLAYER_DATA); } catch (SQLException ex) { mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Chimaera Wing..."); @@ -1234,6 +1243,7 @@ private void checkUpgradeAddUniqueChimaeraWing(final Statement statement) throws private void checkUpgradeAddFishing(final Statement statement) throws SQLException { try { statement.executeQuery("SELECT `fishing` FROM `" + tablePrefix + "skills` LIMIT 1"); + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_FISHING); } catch (SQLException ex) { mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Fishing..."); @@ -1245,6 +1255,7 @@ private void checkUpgradeAddFishing(final Statement statement) throws SQLExcepti private void checkUpgradeAddMobHealthbars(final Statement statement) throws SQLException { try { statement.executeQuery("SELECT `mobhealthbar` FROM `" + tablePrefix + "huds` LIMIT 1"); + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_MOB_HEALTHBARS); } catch (SQLException ex) { mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for mob healthbars..."); @@ -1255,6 +1266,7 @@ private void checkUpgradeAddMobHealthbars(final Statement statement) throws SQLE private void checkUpgradeAddScoreboardTips(final Statement statement) throws SQLException { try { statement.executeQuery("SELECT `scoreboardtips` FROM `" + tablePrefix + "huds` LIMIT 1"); + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_SCOREBOARD_TIPS); } catch (SQLException ex) { mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for scoreboard tips..."); @@ -1283,6 +1295,8 @@ private void checkUpgradeAddSQLIndexes(final Statement statement) { } } } + + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_SQL_INDEXES); } catch (SQLException ex) { printErrors(ex); @@ -1313,6 +1327,8 @@ private void checkUpgradeAddUUIDs(final Statement statement) { statement.executeUpdate("ALTER TABLE `" + tablePrefix + "users` ADD `uuid` varchar(36) NULL DEFAULT NULL"); statement.executeUpdate("ALTER TABLE `" + tablePrefix + "users` ADD UNIQUE INDEX `uuid` (`uuid`) USING BTREE"); } + + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_UUIDS); } catch (SQLException ex) { printErrors(ex); @@ -1379,6 +1395,8 @@ private void checkUpgradeDropPartyNames(final Statement statement) { mcMMO.p.getLogger().info("Removing party name from users table..."); statement.executeUpdate("ALTER TABLE `" + tablePrefix + "users` DROP COLUMN `party`"); } + + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.DROP_SQL_PARTY_NAMES); } catch (SQLException ex) { printErrors(ex); @@ -1414,6 +1432,8 @@ private void checkUpgradeSkillTotal(final Connection connection) throws SQLExcep statement.executeUpdate("ALTER TABLE `" + tablePrefix + "skills` ADD INDEX `idx_total` (`total`) USING BTREE"); connection.commit(); } + + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_SKILL_TOTAL); } catch (SQLException ex) { printErrors(ex); @@ -1445,6 +1465,8 @@ private void checkUpgradeDropSpout(final Statement statement) { mcMMO.p.getLogger().info("Removing Spout HUD type from huds table..."); statement.executeUpdate("ALTER TABLE `" + tablePrefix + "huds` DROP COLUMN `hudtype`"); } + + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.DROP_SPOUT); } catch (SQLException ex) { printErrors(ex); @@ -1557,4 +1579,67 @@ public void resetMobHealthSettings() { tryClose(connection); } } + + private void updateCharacterSet(@NotNull Statement statement) { + //TODO: Could check the tables for being latin1 before executing queries but it seems moot because it is likely the same computational effort + /* + The following columns were set to use latin1 historically (now utf8mb4) + column user in users + column uuid in users + + column mobhealthbar in huds + */ + + //Alter users table + mcMMO.p.getLogger().info("SQL Converting tables from latin1 to utf8mb4"); + + //Update "user" column + try { + mcMMO.p.getLogger().info("Updating user column to new encoding"); + statement.executeUpdate(getUpdateUserInUsersTableSQLQuery()); + + //Update "uuid" column + mcMMO.p.getLogger().info("Updating user column to new encoding"); + statement.executeUpdate(getUpdateUUIDInUsersTableSQLQuery()); + + //Update "mobhealthbar" column + mcMMO.p.getLogger().info("Updating mobhealthbar column to new encoding"); + statement.executeUpdate(getUpdateMobHealthBarInHudsTableSQLQuery()); + + mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.SQL_CHARSET_UTF8MB4); + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @NotNull + private String getUpdateUserInUsersTableSQLQuery() { + return "ALTER TABLE\n" + + " " + tablePrefix + "users\n" + + " CHANGE user user\n" + + " " + USER_VARCHAR + "\n" + + " CHARACTER SET utf8mb4\n" + + " COLLATE utf8mb4_unicode_ci;"; + } + + @NotNull + private String getUpdateUUIDInUsersTableSQLQuery() { + return "ALTER TABLE\n" + + " " + tablePrefix + "users\n" + + " CHANGE uuid uuid\n" + + " " + UUID_VARCHAR + "\n" + + " CHARACTER SET utf8mb4\n" + + " COLLATE utf8mb4_unicode_ci;"; + } + + @NotNull + private String getUpdateMobHealthBarInHudsTableSQLQuery() { + return "ALTER TABLE\n" + + " " + tablePrefix + "huds\n" + + " CHANGE mobhealthbar mobhealthbar\n" + + " " + MOBHEALTHBAR_VARCHAR + "\n" + + " CHARACTER SET utf8mb4\n" + + " COLLATE utf8mb4_unicode_ci;"; + } } diff --git a/src/main/java/com/gmail/nossr50/datatypes/database/UpgradeType.java b/src/main/java/com/gmail/nossr50/datatypes/database/UpgradeType.java index 412af6fbce..13faa75b7f 100644 --- a/src/main/java/com/gmail/nossr50/datatypes/database/UpgradeType.java +++ b/src/main/java/com/gmail/nossr50/datatypes/database/UpgradeType.java @@ -16,5 +16,6 @@ public enum UpgradeType { ADD_UNIQUE_PLAYER_DATA, FIX_SPELLING_NETHERITE_SALVAGE, FIX_SPELLING_NETHERITE_REPAIR, - FIX_NETHERITE_SALVAGE_QUANTITIES + FIX_NETHERITE_SALVAGE_QUANTITIES, + SQL_CHARSET_UTF8MB4 } diff --git a/src/main/java/com/gmail/nossr50/util/upgrade/UpgradeManager.java b/src/main/java/com/gmail/nossr50/util/upgrade/UpgradeManager.java index 082376c0e3..55c747ffd8 100644 --- a/src/main/java/com/gmail/nossr50/util/upgrade/UpgradeManager.java +++ b/src/main/java/com/gmail/nossr50/util/upgrade/UpgradeManager.java @@ -11,7 +11,7 @@ public class UpgradeManager extends ConfigLoader { private final Set setNeededUpgrades; public UpgradeManager() { - super("upgrades.yml"); + super("upgrades_overhaul.yml"); //overhaul is added so we don't have any issues with classic setNeededUpgrades = EnumSet.allOf(UpgradeType.class); diff --git a/src/main/resources/upgrades.yml b/src/main/resources/upgrades_overhaul.yml similarity index 92% rename from src/main/resources/upgrades.yml rename to src/main/resources/upgrades_overhaul.yml index 3a60fb42c1..df0290061a 100644 --- a/src/main/resources/upgrades.yml +++ b/src/main/resources/upgrades_overhaul.yml @@ -11,3 +11,4 @@ Upgrades_Finished: FIX_SPELLING_NETHERITE_REPAIR: false FIX_NETHERITE_SALVAGE_QUANTITIES: false ADD_UUIDS: false + SQL_CHARSET_UTF8MB4: false