From 6603e70aa5abee4dd7eccf2b0b1c7ad829a028e4 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Tue, 23 Aug 2016 21:38:25 +0200 Subject: [PATCH] SONAR-7988 fix mssql collation which was badly defined in 5.x --- .../db/CheckDatabaseCharsetAtStartup.java | 17 +- ...CheckDatabaseCollationDuringMigration.java | 55 ------ .../platformlevel/PlatformLevelStartup.java | 2 - .../db/CheckDatabaseCharsetAtStartupTest.java | 24 ++- ...kDatabaseCollationDuringMigrationTest.java | 74 -------- .../org/sonar/db/charset/CharsetHandler.java | 26 +-- .../java/org/sonar/db/charset/ColumnDef.java | 11 -- .../db/charset/DatabaseCharsetChecker.java | 33 ++-- .../sonar/db/charset/MssqlCharsetHandler.java | 111 +++++------- .../sonar/db/charset/MssqlMetadataReader.java | 73 ++++++++ .../sonar/db/charset/MysqlCharsetHandler.java | 67 +++---- .../db/charset/OracleCharsetHandler.java | 23 +-- .../db/charset/PostgresCharsetHandler.java | 57 +++--- .../db/charset/PostgresMetadataReader.java | 36 ++++ .../org/sonar/db/charset/SqlExecutor.java | 33 +++- .../charset/DatabaseCharsetCheckerTest.java | 31 +--- .../db/charset/MssqlCharsetHandlerTest.java | 170 ++++++++++-------- .../db/charset/MssqlMetadataReaderTest.java | 51 ++++++ .../db/charset/MysqlCharsetHandlerTest.java | 74 +++----- .../db/charset/OracleCharsetHandlerTest.java | 49 ++--- .../charset/PostgresCharsetHandlerTest.java | 94 ++++++---- .../charset/PostgresMetadataReaderTest.java | 52 ++++++ .../sonar/db/charset/SelectExecutorTest.java | 2 +- .../org/sonar/db/charset/SqlExecutorTest.java | 4 +- 24 files changed, 608 insertions(+), 561 deletions(-) delete mode 100644 server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigration.java delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigrationTest.java create mode 100644 sonar-db/src/main/java/org/sonar/db/charset/MssqlMetadataReader.java create mode 100644 sonar-db/src/main/java/org/sonar/db/charset/PostgresMetadataReader.java create mode 100644 sonar-db/src/test/java/org/sonar/db/charset/MssqlMetadataReaderTest.java create mode 100644 sonar-db/src/test/java/org/sonar/db/charset/PostgresMetadataReaderTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartup.java b/server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartup.java index ee1d20ca567e..9d61afdd0bcb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartup.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartup.java @@ -23,8 +23,6 @@ import org.sonar.api.platform.ServerUpgradeStatus; import org.sonar.db.charset.DatabaseCharsetChecker; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; - /** * Checks charset of all existing database columns at startup, before executing db migrations. This requires * to be defined in platform level 2 ({@link org.sonar.server.platform.platformlevel.PlatformLevel2}). @@ -41,7 +39,13 @@ public CheckDatabaseCharsetAtStartup(ServerUpgradeStatus upgradeStatus, Database @Override public void start() { - check(); + DatabaseCharsetChecker.State state = DatabaseCharsetChecker.State.STARTUP; + if (upgradeStatus.isUpgraded()) { + state = DatabaseCharsetChecker.State.UPGRADE; + } else if (upgradeStatus.isFreshInstall()) { + state = DatabaseCharsetChecker.State.FRESH_INSTALL; + } + charsetChecker.check(state); } @Override @@ -49,11 +53,4 @@ public void stop() { // do nothing } - protected final void check() { - if (upgradeStatus.isFreshInstall()) { - charsetChecker.check(ENFORCE_UTF8); - } else if (!upgradeStatus.isUpgraded()) { - charsetChecker.check(); - } - } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigration.java b/server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigration.java deleted file mode 100644 index f78b0460be74..000000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigration.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.platform.db; - -import org.picocontainer.Startable; -import org.sonar.api.platform.ServerUpgradeStatus; -import org.sonar.db.charset.DatabaseCharsetChecker; - -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.AUTO_REPAIR_COLLATION; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; - -/** - * Checks charset of all database columns when at least one db migration has been executed. - */ -public class CheckDatabaseCollationDuringMigration implements Startable { - - private final ServerUpgradeStatus upgradeStatus; - private final DatabaseCharsetChecker charsetChecker; - - public CheckDatabaseCollationDuringMigration(ServerUpgradeStatus upgradeStatus, DatabaseCharsetChecker charsetChecker) { - this.upgradeStatus = upgradeStatus; - this.charsetChecker = charsetChecker; - } - - @Override - public void start() { - if (upgradeStatus.isFreshInstall()) { - charsetChecker.check(ENFORCE_UTF8, AUTO_REPAIR_COLLATION); - } else if (upgradeStatus.isUpgraded()) { - charsetChecker.check(AUTO_REPAIR_COLLATION); - } - } - - @Override - public void stop() { - // do nothing - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java index 99010504bd18..09b1365e828c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java @@ -23,7 +23,6 @@ import org.sonar.server.es.IndexerStartupTask; import org.sonar.server.issue.filter.RegisterIssueFilters; import org.sonar.server.platform.ServerLifecycleNotifier; -import org.sonar.server.platform.db.CheckDatabaseCollationDuringMigration; import org.sonar.server.platform.web.RegisterServletFilters; import org.sonar.server.qualitygate.RegisterQualityGates; import org.sonar.server.qualityprofile.RegisterQualityProfiles; @@ -53,7 +52,6 @@ protected void configureLevel() { ServerLifecycleNotifier.class); addIfStartupLeader( - CheckDatabaseCollationDuringMigration.class, IndexerStartupTask.class, RegisterMetrics.class, RegisterQualityGates.class, diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartupTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartupTest.java index e3711be8417b..740fce13b1f6 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartupTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCharsetAtStartupTest.java @@ -27,13 +27,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; public class CheckDatabaseCharsetAtStartupTest { - ServerUpgradeStatus upgradeStatus = mock(ServerUpgradeStatus.class); - DatabaseCharsetChecker charsetChecker = mock(DatabaseCharsetChecker.class); - CheckDatabaseCharsetAtStartup underTest = new CheckDatabaseCharsetAtStartup(upgradeStatus, charsetChecker); + private ServerUpgradeStatus upgradeStatus = mock(ServerUpgradeStatus.class); + private DatabaseCharsetChecker charsetChecker = mock(DatabaseCharsetChecker.class); + private CheckDatabaseCharsetAtStartup underTest = new CheckDatabaseCharsetAtStartup(upgradeStatus, charsetChecker); @After public void tearDown() { @@ -41,20 +40,29 @@ public void tearDown() { } @Test - public void enforce_utf8_if_fresh_install() { + public void test_fresh_install() { when(upgradeStatus.isFreshInstall()).thenReturn(true); underTest.start(); - verify(charsetChecker).check(ENFORCE_UTF8); + verify(charsetChecker).check(DatabaseCharsetChecker.State.FRESH_INSTALL); } @Test - public void do_not_enforce_utf8_and_do_not_repair_at_startup_if_not_fresh_install() { + public void test_upgrade() { + when(upgradeStatus.isUpgraded()).thenReturn(true); + + underTest.start(); + + verify(charsetChecker).check(DatabaseCharsetChecker.State.UPGRADE); + } + + @Test + public void test_regular_startup() { when(upgradeStatus.isFreshInstall()).thenReturn(false); underTest.start(); - verify(charsetChecker).check(); + verify(charsetChecker).check(DatabaseCharsetChecker.State.STARTUP); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigrationTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigrationTest.java deleted file mode 100644 index 043750452ecb..000000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/db/CheckDatabaseCollationDuringMigrationTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.platform.db; - -import org.junit.After; -import org.junit.Test; -import org.sonar.api.platform.ServerUpgradeStatus; -import org.sonar.db.charset.DatabaseCharsetChecker; -import org.sonar.server.platform.db.CheckDatabaseCollationDuringMigration; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.AUTO_REPAIR_COLLATION; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; - -public class CheckDatabaseCollationDuringMigrationTest { - ServerUpgradeStatus upgradeStatus = mock(ServerUpgradeStatus.class); - DatabaseCharsetChecker charsetChecker = mock(DatabaseCharsetChecker.class); - CheckDatabaseCollationDuringMigration underTest = new CheckDatabaseCollationDuringMigration(upgradeStatus, charsetChecker); - - @After - public void tearDown() { - underTest.stop(); - } - - @Test - public void enforce_utf8_and_optionally_repair_collation_if_fresh_install() { - when(upgradeStatus.isFreshInstall()).thenReturn(true); - - underTest.start(); - - verify(charsetChecker).check(ENFORCE_UTF8, AUTO_REPAIR_COLLATION); - } - - @Test - public void repair_collation_but_do_not_enforce_utf8_if_db_upgrade() { - when(upgradeStatus.isFreshInstall()).thenReturn(false); - when(upgradeStatus.isUpgraded()).thenReturn(true); - - underTest.start(); - - verify(charsetChecker).check(AUTO_REPAIR_COLLATION); - } - - @Test - public void do_nothing_if_no_db_changes() { - when(upgradeStatus.isFreshInstall()).thenReturn(false); - when(upgradeStatus.isUpgraded()).thenReturn(false); - - underTest.start(); - - verifyZeroInteractions(charsetChecker); - } - -} diff --git a/sonar-db/src/main/java/org/sonar/db/charset/CharsetHandler.java b/sonar-db/src/main/java/org/sonar/db/charset/CharsetHandler.java index df8d2d4fa3ca..de27d9925e03 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/CharsetHandler.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/CharsetHandler.java @@ -21,9 +21,6 @@ import java.sql.Connection; import java.sql.SQLException; -import java.util.List; -import java.util.Set; -import javax.annotation.CheckForNull; abstract class CharsetHandler { @@ -35,31 +32,10 @@ protected CharsetHandler(SqlExecutor selectExecutor) { this.selectExecutor = selectExecutor; } - abstract void handle(Connection connection, Set flags) throws SQLException; + abstract void handle(Connection connection, DatabaseCharsetChecker.State state) throws SQLException; protected SqlExecutor getSqlExecutor() { return selectExecutor; } - @CheckForNull - protected final String selectSingleString(Connection connection, String sql) throws SQLException { - String[] cols = selectSingleRow(connection, sql, new SqlExecutor.StringsConverter(1)); - return cols == null ? null : cols[0]; - } - - @CheckForNull - protected final T selectSingleRow(Connection connection, String sql, SqlExecutor.RowConverter rowConverter) throws SQLException { - List rows = select(connection, sql, rowConverter); - if (rows.isEmpty()) { - return null; - } - if (rows.size() == 1) { - return rows.get(0); - } - throw new IllegalStateException("Expecting only one result for [" + sql + "]"); - } - - protected final List select(Connection connection, String sql, SqlExecutor.RowConverter rowConverter) throws SQLException { - return selectExecutor.executeSelect(connection, sql, rowConverter); - } } diff --git a/sonar-db/src/main/java/org/sonar/db/charset/ColumnDef.java b/sonar-db/src/main/java/org/sonar/db/charset/ColumnDef.java index 461bf32365b2..02213499e7f6 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/ColumnDef.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/ColumnDef.java @@ -19,11 +19,9 @@ */ package org.sonar.db.charset; -import com.google.common.base.Predicate; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Locale; -import javax.annotation.Nonnull; import javax.annotation.concurrent.Immutable; import org.sonar.db.version.DatabaseVersion; @@ -97,13 +95,4 @@ public ColumnDef convert(ResultSet rs) throws SQLException { rs.getString(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getString(5), rs.getLong(6), nullable); } } - - public enum IsInSonarQubeTablePredicate implements Predicate { - INSTANCE; - - @Override - public boolean apply(@Nonnull ColumnDef input) { - return input.isInSonarQubeTable(); - } - } } diff --git a/sonar-db/src/main/java/org/sonar/db/charset/DatabaseCharsetChecker.java b/sonar-db/src/main/java/org/sonar/db/charset/DatabaseCharsetChecker.java index 2e89e2d3a97a..5a91685dc2ee 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/DatabaseCharsetChecker.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/DatabaseCharsetChecker.java @@ -31,9 +31,6 @@ import org.sonar.db.dialect.Oracle; import org.sonar.db.dialect.PostgreSql; -import static com.google.common.collect.Sets.immutableEnumSet; -import static java.util.Arrays.asList; - /** * On fresh installations, checks that all db columns are UTF8. On all installations on MySQL or MSSQL, * whatever fresh or upgrade, fixes case-insensitive columns by converting them to @@ -43,30 +40,28 @@ */ public class DatabaseCharsetChecker { - public enum Flag { - ENFORCE_UTF8, AUTO_REPAIR_COLLATION + public enum State { + FRESH_INSTALL, UPGRADE, STARTUP } private final Database db; - private final SqlExecutor selectExecutor; + private final SqlExecutor sqlExecutor; public DatabaseCharsetChecker(Database db) { this(db, new SqlExecutor()); } @VisibleForTesting - DatabaseCharsetChecker(Database db, SqlExecutor selectExecutor) { + DatabaseCharsetChecker(Database db, SqlExecutor sqlExecutor) { this.db = db; - this.selectExecutor = selectExecutor; + this.sqlExecutor = sqlExecutor; } - public void check(Flag... flags) { - try { - try (Connection connection = db.getDataSource().getConnection()) { - CharsetHandler handler = getHandler(db.getDialect()); - if (handler != null) { - handler.handle(connection, immutableEnumSet(asList(flags))); - } + public void check(State state) { + try (Connection connection = db.getDataSource().getConnection()) { + CharsetHandler handler = getHandler(db.getDialect()); + if (handler != null) { + handler.handle(connection, state); } } catch (SQLException e) { throw new IllegalStateException(e); @@ -81,13 +76,13 @@ CharsetHandler getHandler(Dialect dialect) { // nothing to check return null; case Oracle.ID: - return new OracleCharsetHandler(selectExecutor); + return new OracleCharsetHandler(sqlExecutor); case PostgreSql.ID: - return new PostgresCharsetHandler(selectExecutor); + return new PostgresCharsetHandler(sqlExecutor, new PostgresMetadataReader(sqlExecutor)); case MySql.ID: - return new MysqlCharsetHandler(selectExecutor); + return new MysqlCharsetHandler(sqlExecutor); case MsSql.ID: - return new MssqlCharsetHandler(selectExecutor); + return new MssqlCharsetHandler(sqlExecutor, new MssqlMetadataReader(sqlExecutor)); default: throw new IllegalArgumentException("Database not supported: " + dialect.getId()); } diff --git a/sonar-db/src/main/java/org/sonar/db/charset/MssqlCharsetHandler.java b/sonar-db/src/main/java/org/sonar/db/charset/MssqlCharsetHandler.java index 6b3cef60d1fa..818714966dc1 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/MssqlCharsetHandler.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/MssqlCharsetHandler.java @@ -20,21 +20,17 @@ package org.sonar.db.charset; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; +import java.util.stream.Collectors; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import static com.google.common.collect.FluentIterable.from; import static java.lang.String.format; import static org.apache.commons.lang.StringUtils.containsIgnoreCase; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.AUTO_REPAIR_COLLATION; class MssqlCharsetHandler extends CharsetHandler { @@ -46,104 +42,80 @@ class MssqlCharsetHandler extends CharsetHandler { private static final String BIN = "BIN"; private static final String BIN2 = "BIN2"; - protected MssqlCharsetHandler(SqlExecutor selectExecutor) { + private final MssqlMetadataReader metadata; + + MssqlCharsetHandler(SqlExecutor selectExecutor, MssqlMetadataReader metadataReader) { super(selectExecutor); + this.metadata = metadataReader; } @Override - void handle(Connection connection, Set flags) throws SQLException { - logInit(flags); + void handle(Connection connection, DatabaseCharsetChecker.State state) throws SQLException { + expectCaseSensitiveDefaultCollation(connection); + if (state == DatabaseCharsetChecker.State.UPGRADE || state == DatabaseCharsetChecker.State.STARTUP) { + repairColumns(connection); + } + } + + private void expectCaseSensitiveDefaultCollation(Connection connection) throws SQLException { + LOGGER.info("Verify that database collation is case-sensitive and accent-sensitive"); + String defaultCollation = metadata.getDefaultCollation(connection); + + if (!isCollationCorrect(defaultCollation)) { + String fixedCollation = toCaseSensitive(defaultCollation); + throw MessageException.of(format( + "Database collation must be case-sensitive and accent-sensitive. It is %s but should be %s.", defaultCollation, fixedCollation)); + } + } + + private void repairColumns(Connection connection) throws SQLException { + String defaultCollation = metadata.getDefaultCollation(connection); // All VARCHAR columns are returned. No need to check database general collation. // Example of row: // issues | kee | Latin1_General_CS_AS or Latin1_General_100_CI_AS_KS_WS - Set errors = new LinkedHashSet<>(); - List columns = select(connection, - ColumnDef.SELECT_COLUMNS + - "FROM [INFORMATION_SCHEMA].[COLUMNS] " + - "WHERE collation_name is not null " + - "ORDER BY table_name,column_name", - ColumnDef.ColumnDefRowConverter.INSTANCE); - for (ColumnDef column : from(columns).filter(ColumnDef.IsInSonarQubeTablePredicate.INSTANCE)) { - if (!isCollationCorrect(column)) { - if (flags.contains(AUTO_REPAIR_COLLATION)) { - repairColumnCollation(connection, column); - } else { - errors.add(format("%s.%s", column.getTable(), column.getColumn())); - } + List columns = metadata.getColumnDefs(connection); + for (ColumnDef column : columns.stream().filter(ColumnDef::isInSonarQubeTable).collect(Collectors.toList())) { + String collation = column.getCollation(); + if (!isCollationCorrect(collation)) { + repairColumnCollation(connection, column, toCaseSensitive(collation)); + } else if ("Latin1_General_CS_AS".equals(collation) && !collation.equals(defaultCollation)) { + repairColumnCollation(connection, column, defaultCollation); } } - - if (!errors.isEmpty()) { - throw MessageException.of(format("Case-sensitive and accent-sensitive collation is required for database columns [%s]", - Joiner.on(", ").join(errors))); - } } /** - * Collation is correct if is contains {@link #CASE_SENSITIVE_ACCENT_SENSITIVE} or {@link #BIN} or {@link #BIN2}. + * Collation is correct if contains {@link #CASE_SENSITIVE_ACCENT_SENSITIVE} or {@link #BIN} or {@link #BIN2}. */ - private static boolean isCollationCorrect(ColumnDef column) { - String collation = column.getCollation(); + private static boolean isCollationCorrect(String collation) { return containsIgnoreCase(collation, CASE_SENSITIVE_ACCENT_SENSITIVE) || containsIgnoreCase(collation, BIN) || containsIgnoreCase(collation, BIN2); } - private static void logInit(Set flags) { - if (flags.contains(AUTO_REPAIR_COLLATION)) { - LOGGER.info("Repair case-insensitive or accent-insensitive database columns"); - } else { - LOGGER.info("Verify that database columns are case-sensitive and accent-sensitive"); - } - } - - private void repairColumnCollation(Connection connection, ColumnDef column) throws SQLException { + private void repairColumnCollation(Connection connection, ColumnDef column, String expectedCollation) throws SQLException { // 1. select the indices defined on this column - String selectIndicesSql = format("SELECT I.name as index_name, I.is_unique as unik, IndexedColumns " + - " FROM sys.indexes I " + - " JOIN sys.tables T ON T.Object_id = I.Object_id " + - " JOIN (SELECT * FROM ( " + - " SELECT IC2.object_id, IC2.index_id, " + - " STUFF((SELECT ' ,' + C.name " + - " FROM sys.index_columns IC1 " + - " JOIN sys.columns C " + - " ON C.object_id = IC1.object_id " + - " AND C.column_id = IC1.column_id " + - " AND IC1.is_included_column = 0 " + - " WHERE IC1.object_id = IC2.object_id " + - " AND IC1.index_id = IC2.index_id " + - " GROUP BY IC1.object_id,C.name,index_id " + - " ORDER BY MAX(IC1.key_ordinal) " + - " FOR XML PATH('')), 1, 2, '') IndexedColumns " + - " FROM sys.index_columns IC2 " + - " GROUP BY IC2.object_id ,IC2.index_id) tmp1 )tmp2 " + - " ON I.object_id = tmp2.object_id AND I.Index_id = tmp2.index_id " + - " WHERE I.is_primary_key = 0 AND I.is_unique_constraint = 0 " + - " and T.name =('%s') " + - " and CHARINDEX ('%s',IndexedColumns)>0", column.getTable(), column.getColumn()); - List indices = getSqlExecutor().executeSelect(connection, selectIndicesSql, ColumnIndexConverter.INSTANCE); + List indices = metadata.getColumnIndices(connection, column); // 2. drop indices for (ColumnIndex index : indices) { - getSqlExecutor().executeUpdate(connection, format("DROP INDEX %s.%s", column.getTable(), index.name)); + getSqlExecutor().executeDdl(connection, format("DROP INDEX %s.%s", column.getTable(), index.name)); } // 3. alter collation of column - String csCollation = toCaseSensitive(column.getCollation()); - String nullability = column.isNullable() ? "NULL" : "NOT NULL"; String size = column.getSize() >= 0 ? String.valueOf(column.getSize()) : "max"; String alterSql = format("ALTER TABLE %s ALTER COLUMN %s %s(%s) COLLATE %s %s", - column.getTable(), column.getColumn(), column.getDataType(), size, csCollation, nullability); - LOGGER.info("Changing collation of column [{}.{}] from {} to {} | sql=", column.getTable(), column.getColumn(), column.getCollation(), csCollation, alterSql); - getSqlExecutor().executeUpdate(connection, alterSql); + column.getTable(), column.getColumn(), column.getDataType(), size, expectedCollation, nullability); + LOGGER.info("Changing collation of column [{}.{}] from {} to {} | sql=", column.getTable(), column.getColumn(), column.getCollation(), expectedCollation, alterSql); + getSqlExecutor().executeDdl(connection, alterSql); // 4. re-create indices for (ColumnIndex index : indices) { String uniqueSql = index.unique ? "UNIQUE" : ""; String createIndexSql = format("CREATE %s INDEX %s ON %s (%s)", uniqueSql, index.name, column.getTable(), index.csvColumns); - getSqlExecutor().executeUpdate(connection, createIndexSql); + getSqlExecutor().executeDdl(connection, createIndexSql); } } @@ -177,4 +149,5 @@ public ColumnIndex convert(ResultSet rs) throws SQLException { return new ColumnIndex(rs.getString(1), rs.getBoolean(2), rs.getString(3)); } } + } diff --git a/sonar-db/src/main/java/org/sonar/db/charset/MssqlMetadataReader.java b/sonar-db/src/main/java/org/sonar/db/charset/MssqlMetadataReader.java new file mode 100644 index 000000000000..0f7aa18afe72 --- /dev/null +++ b/sonar-db/src/main/java/org/sonar/db/charset/MssqlMetadataReader.java @@ -0,0 +1,73 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.charset; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; + +import static java.lang.String.format; + +public class MssqlMetadataReader { + private final SqlExecutor sqlExecutor; + + public MssqlMetadataReader(SqlExecutor sqlExecutor) { + this.sqlExecutor = sqlExecutor; + } + + public String getDefaultCollation(Connection connection) throws SQLException { + return sqlExecutor.selectSingleString(connection, "SELECT CONVERT(VARCHAR, DATABASEPROPERTYEX(DB_NAME(), 'Collation'))"); + } + + public List getColumnDefs(Connection connection) throws SQLException { + return sqlExecutor.select(connection, + ColumnDef.SELECT_COLUMNS + + "FROM [INFORMATION_SCHEMA].[COLUMNS] " + + "WHERE collation_name is not null " + + "ORDER BY table_name,column_name", + ColumnDef.ColumnDefRowConverter.INSTANCE); + } + + public List getColumnIndices(Connection connection, ColumnDef column) throws SQLException { + String selectIndicesSql = format("SELECT I.name as index_name, I.is_unique as unik, IndexedColumns " + + " FROM sys.indexes I " + + " JOIN sys.tables T ON T.Object_id = I.Object_id " + + " JOIN (SELECT * FROM ( " + + " SELECT IC2.object_id, IC2.index_id, " + + " STUFF((SELECT ' ,' + C.name " + + " FROM sys.index_columns IC1 " + + " JOIN sys.columns C " + + " ON C.object_id = IC1.object_id " + + " AND C.column_id = IC1.column_id " + + " AND IC1.is_included_column = 0 " + + " WHERE IC1.object_id = IC2.object_id " + + " AND IC1.index_id = IC2.index_id " + + " GROUP BY IC1.object_id,C.name,index_id " + + " ORDER BY MAX(IC1.key_ordinal) " + + " FOR XML PATH('')), 1, 2, '') IndexedColumns " + + " FROM sys.index_columns IC2 " + + " GROUP BY IC2.object_id ,IC2.index_id) tmp1 )tmp2 " + + " ON I.object_id = tmp2.object_id AND I.Index_id = tmp2.index_id " + + " WHERE I.is_primary_key = 0 AND I.is_unique_constraint = 0 " + + " and T.name =('%s') " + + " and CHARINDEX ('%s',IndexedColumns)>0", column.getTable(), column.getColumn()); + return sqlExecutor.select(connection, selectIndicesSql, MssqlCharsetHandler.ColumnIndexConverter.INSTANCE); + } +} diff --git a/sonar-db/src/main/java/org/sonar/db/charset/MysqlCharsetHandler.java b/sonar-db/src/main/java/org/sonar/db/charset/MysqlCharsetHandler.java index 4f003aba7356..2190e6a3bb12 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/MysqlCharsetHandler.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/MysqlCharsetHandler.java @@ -19,73 +19,55 @@ */ package org.sonar.db.charset; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; import java.sql.Connection; import java.sql.SQLException; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; -import org.sonar.api.utils.MessageException; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import static com.google.common.collect.FluentIterable.from; import static java.lang.String.format; -import static org.apache.commons.lang.StringUtils.containsIgnoreCase; import static org.apache.commons.lang.StringUtils.endsWithIgnoreCase; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.AUTO_REPAIR_COLLATION; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; class MysqlCharsetHandler extends CharsetHandler { private static final Logger LOGGER = Loggers.get(MysqlCharsetHandler.class); private static final String TYPE_LONGTEXT = "longtext"; - protected MysqlCharsetHandler(SqlExecutor selectExecutor) { + MysqlCharsetHandler(SqlExecutor selectExecutor) { super(selectExecutor); } @Override - void handle(Connection connection, Set flags) throws SQLException { - logInit(flags); - checkCollation(connection, flags); - } - - private static void logInit(Set flags) { - if (flags.contains(AUTO_REPAIR_COLLATION)) { - LOGGER.info("Repair case-insensitive database columns"); - } else if (flags.contains(ENFORCE_UTF8)) { - LOGGER.info("Verify that database collation is UTF8"); - } else { - LOGGER.info("Verify that database collation is case-sensitive"); + void handle(Connection connection, DatabaseCharsetChecker.State state) throws SQLException { + // all the VARCHAR columns have always been created with UTF8 charset on mysql + // (since SonarQube 2.12 to be precise). The default charset does not require + // to be UTF8. It is not used. No need to verify it. + // Still if a column has been accidentally created with a case-insensitive collation, + // then we can repair it by moving to the same case-sensitive collation. That should + // never occur. + if (state == DatabaseCharsetChecker.State.UPGRADE) { + repairCaseInsensitiveColumns(connection); } } - private void checkCollation(Connection connection, Set flags) throws SQLException { + private void repairCaseInsensitiveColumns(Connection connection) throws SQLException { // All VARCHAR columns are returned. No need to check database general collation. // Example of row: // issues | kee | utf8 | utf8_bin - List columns = select(connection, + List columns = getSqlExecutor().select(connection, ColumnDef.SELECT_COLUMNS + "FROM INFORMATION_SCHEMA.columns " + - "WHERE table_schema=database() and character_set_name is not null and collation_name is not null", ColumnDef.ColumnDefRowConverter.INSTANCE); - Set errors = new LinkedHashSet<>(); - for (ColumnDef column : from(columns).filter(ColumnDef.IsInSonarQubeTablePredicate.INSTANCE)) { - if (flags.contains(ENFORCE_UTF8) && !containsIgnoreCase(column.getCharset(), UTF8)) { - errors.add(format("%s.%s", column.getTable(), column.getColumn())); - } - if (endsWithIgnoreCase(column.getCollation(), "_ci")) { - if (flags.contains(AUTO_REPAIR_COLLATION)) { - repairCaseInsensitiveColumn(connection, column); - } else { - errors.add(format("%s.%s", column.getTable(), column.getColumn())); - } - } - } - if (!errors.isEmpty()) { - throw MessageException.of(format("UTF8 case-sensitive collation is required for database columns [%s]", Joiner.on(", ").join(errors))); + "WHERE table_schema=database() and character_set_name is not null and collation_name is not null", + ColumnDef.ColumnDefRowConverter.INSTANCE); + + List invalidColumns = columns.stream() + .filter(ColumnDef::isInSonarQubeTable) + .filter(column -> endsWithIgnoreCase(column.getCollation(), "_ci")) + .collect(Collectors.toList()); + for (ColumnDef column : invalidColumns) { + repairCaseInsensitiveColumn(connection, column); } } @@ -98,11 +80,10 @@ private void repairCaseInsensitiveColumn(Connection connection, ColumnDef column String alterSql = format("ALTER TABLE %s MODIFY %s %s CHARACTER SET '%s' COLLATE '%s' %s", column.getTable(), column.getColumn(), type, column.getCharset(), csCollation, nullability); LOGGER.info("Changing collation of column [{}.{}] from {} to {} | sql={}", column.getTable(), column.getColumn(), column.getCollation(), csCollation, alterSql); - getSqlExecutor().executeUpdate(connection, alterSql); + getSqlExecutor().executeDdl(connection, alterSql); } - @VisibleForTesting - static String toCaseSensitive(String caseInsensitiveCollation) { + private static String toCaseSensitive(String caseInsensitiveCollation) { // Example: big5_chinese_ci becomes big5_bin // Full list of collations is available with SQL request "show collation" return StringUtils.substringBefore(caseInsensitiveCollation, "_") + "_bin"; diff --git a/sonar-db/src/main/java/org/sonar/db/charset/OracleCharsetHandler.java b/sonar-db/src/main/java/org/sonar/db/charset/OracleCharsetHandler.java index 1ff377175cf6..09f855958e61 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/OracleCharsetHandler.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/OracleCharsetHandler.java @@ -21,32 +21,35 @@ import java.sql.Connection; import java.sql.SQLException; -import java.util.Set; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.log.Loggers; import static java.lang.String.format; import static org.apache.commons.lang.StringUtils.containsIgnoreCase; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; class OracleCharsetHandler extends CharsetHandler { - protected OracleCharsetHandler(SqlExecutor selectExecutor) { + OracleCharsetHandler(SqlExecutor selectExecutor) { super(selectExecutor); } @Override - public void handle(Connection connection, Set flags) throws SQLException { - // Oracle does not allow to override character set on tables. Only global charset is verified. - if (flags.contains(ENFORCE_UTF8)) { + public void handle(Connection connection, DatabaseCharsetChecker.State state) throws SQLException { + // Charset is a global setting on Oracle, it can't be set on a specified schema with a + // different value. To not block users who already have a SonarQube schema, charset + // is verified only on fresh installs but not on upgrades. Let's hope they won't face + // any errors related to charset if they didn't follow the UTF8 requirement when creating + // the schema in previous SonarQube versions. + if (state == DatabaseCharsetChecker.State.FRESH_INSTALL) { Loggers.get(getClass()).info("Verify that database charset is UTF8"); - checkUtf8(connection); + expectUtf8(connection); } } - private void checkUtf8(Connection connection) throws SQLException { - String charset = selectSingleString(connection, "select value from nls_database_parameters where parameter='NLS_CHARACTERSET'"); - String sort = selectSingleString(connection, "select value from nls_database_parameters where parameter='NLS_SORT'"); + private void expectUtf8(Connection connection) throws SQLException { + // Oracle does not allow to override character set on tables. Only global charset is verified. + String charset = getSqlExecutor().selectSingleString(connection, "select value from nls_database_parameters where parameter='NLS_CHARACTERSET'"); + String sort = getSqlExecutor().selectSingleString(connection, "select value from nls_database_parameters where parameter='NLS_SORT'"); if (!containsIgnoreCase(charset, UTF8) || !"BINARY".equalsIgnoreCase(sort)) { throw MessageException.of(format("Oracle must be have UTF8 charset and BINARY sort. NLS_CHARACTERSET is %s and NLS_SORT is %s.", charset, sort)); } diff --git a/sonar-db/src/main/java/org/sonar/db/charset/PostgresCharsetHandler.java b/sonar-db/src/main/java/org/sonar/db/charset/PostgresCharsetHandler.java index 81cd5bcc1a36..69539cfb74e7 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/PostgresCharsetHandler.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/PostgresCharsetHandler.java @@ -19,64 +19,75 @@ */ package org.sonar.db.charset; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import java.sql.Connection; import java.sql.SQLException; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import org.apache.commons.lang.StringUtils; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.log.Loggers; import static java.lang.String.format; import static org.apache.commons.lang.StringUtils.containsIgnoreCase; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; +import static org.apache.commons.lang.StringUtils.isBlank; class PostgresCharsetHandler extends CharsetHandler { - protected PostgresCharsetHandler(SqlExecutor selectExecutor) { + private final PostgresMetadataReader metadata; + + PostgresCharsetHandler(SqlExecutor selectExecutor, PostgresMetadataReader metadata) { super(selectExecutor); + this.metadata = metadata; } @Override - void handle(Connection connection, Set flags) throws SQLException { - // PostgreSQL does not support case-insensitive collations. Only charset must be verified. - if (flags.contains(ENFORCE_UTF8)) { - Loggers.get(getClass()).info("Verify that database collation supports UTF8"); - checkUtf8(connection); + void handle(Connection connection, DatabaseCharsetChecker.State state) throws SQLException { + // PostgreSQL does not have concept of case-sensitive collation. Only charset ("encoding" in postgresql terminology) + // must be verified. + expectUtf8AsDefault(connection); + + if (state == DatabaseCharsetChecker.State.UPGRADE || state == DatabaseCharsetChecker.State.STARTUP) { + // no need to check columns on fresh installs... as they are not supposed to exist! + expectUtf8Columns(connection); + } + } + + private void expectUtf8AsDefault(Connection connection) throws SQLException { + Loggers.get(getClass()).info("Verify that database charset supports UTF8"); + String collation = metadata.getDefaultCharset(connection); + if (!containsIgnoreCase(collation, UTF8)) { + throw MessageException.of(format("Database charset is %s. It must support UTF8.", collation)); } } - private void checkUtf8(Connection connection) throws SQLException { - // Character set is defined globally and can be overridden on each column. - // This request returns all VARCHAR columns. Collation may be empty. + private void expectUtf8Columns(Connection connection) throws SQLException { + // Charset is defined globally and can be overridden on each column. + // This request returns all VARCHAR columns. Charset may be empty. // Examples: // issues | key | '' // projects | name | utf8 - List rows = select(connection, "select table_name, column_name, collation_name " + + List rows = getSqlExecutor().select(connection, "select table_name, column_name, collation_name " + "from information_schema.columns " + "where table_schema='public' " + "and udt_name='varchar' " + "order by table_name, column_name", new SqlExecutor.StringsConverter(3 /* columns returned by SELECT */)); - boolean mustCheckGlobalCollation = false; Set errors = new LinkedHashSet<>(); for (String[] row : rows) { - if (StringUtils.isBlank(row[2])) { - mustCheckGlobalCollation = true; - } else if (!containsIgnoreCase(row[2], UTF8)) { + if (!isBlank(row[2]) && !containsIgnoreCase(row[2], UTF8)) { errors.add(format("%s.%s", row[0], row[1])); } } - if (mustCheckGlobalCollation) { - String charset = selectSingleString(connection, "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()"); - if (!containsIgnoreCase(charset, UTF8)) { - throw MessageException.of(format("Database collation is %s. It must support UTF8.", charset)); - } - } if (!errors.isEmpty()) { - throw MessageException.of(format("Database columns [%s] must support UTF8 collation.", Joiner.on(", ").join(errors))); + throw MessageException.of(format("Database columns [%s] must have UTF8 charset.", Joiner.on(", ").join(errors))); } } + + @VisibleForTesting + PostgresMetadataReader getMetadata() { + return metadata; + } + } diff --git a/sonar-db/src/main/java/org/sonar/db/charset/PostgresMetadataReader.java b/sonar-db/src/main/java/org/sonar/db/charset/PostgresMetadataReader.java new file mode 100644 index 000000000000..eecca8d1c01c --- /dev/null +++ b/sonar-db/src/main/java/org/sonar/db/charset/PostgresMetadataReader.java @@ -0,0 +1,36 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.charset; + +import java.sql.Connection; +import java.sql.SQLException; + +public class PostgresMetadataReader { + + private final SqlExecutor sqlExecutor; + + public PostgresMetadataReader(SqlExecutor sqlExecutor) { + this.sqlExecutor = sqlExecutor; + } + + public String getDefaultCharset(Connection connection) throws SQLException { + return sqlExecutor.selectSingleString(connection, "select pg_encoding_to_char(encoding) from pg_database where datname = current_database()"); + } +} diff --git a/sonar-db/src/main/java/org/sonar/db/charset/SqlExecutor.java b/sonar-db/src/main/java/org/sonar/db/charset/SqlExecutor.java index 6d0e60a8a0fb..f9e56cd75964 100644 --- a/sonar-db/src/main/java/org/sonar/db/charset/SqlExecutor.java +++ b/sonar-db/src/main/java/org/sonar/db/charset/SqlExecutor.java @@ -23,13 +23,15 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import javax.annotation.CheckForNull; import org.sonar.db.DatabaseUtils; public class SqlExecutor { - public List executeSelect(Connection connection, String sql, RowConverter rowConverter) throws SQLException { + public List select(Connection connection, String sql, RowConverter rowConverter) throws SQLException { PreparedStatement stmt = null; ResultSet rs = null; try { @@ -47,16 +49,31 @@ public List executeSelect(Connection connection, String sql, RowConverter } } - public void executeUpdate(Connection connection, String sql) throws SQLException { - PreparedStatement stmt = null; - try { - stmt = connection.prepareStatement(sql); - stmt.executeUpdate(); - } finally { - DatabaseUtils.closeQuietly(stmt); + public void executeDdl(Connection connection, String sql) throws SQLException { + try (Statement stmt = connection.createStatement()) { + stmt.execute(sql); + } + } + + @CheckForNull + public final String selectSingleString(Connection connection, String sql) throws SQLException { + String[] cols = selectSingleRow(connection, sql, new SqlExecutor.StringsConverter(1)); + return cols == null ? null : cols[0]; + } + + @CheckForNull + public final T selectSingleRow(Connection connection, String sql, SqlExecutor.RowConverter rowConverter) throws SQLException { + List rows = select(connection, sql, rowConverter); + if (rows.isEmpty()) { + return null; + } + if (rows.size() == 1) { + return rows.get(0); } + throw new IllegalStateException("Expecting only one result for [" + sql + "]"); } + @FunctionalInterface public interface RowConverter { T convert(ResultSet rs) throws SQLException; } diff --git a/sonar-db/src/test/java/org/sonar/db/charset/DatabaseCharsetCheckerTest.java b/sonar-db/src/test/java/org/sonar/db/charset/DatabaseCharsetCheckerTest.java index 41c87e6b28fc..d707ff9ee0ba 100644 --- a/sonar-db/src/test/java/org/sonar/db/charset/DatabaseCharsetCheckerTest.java +++ b/sonar-db/src/test/java/org/sonar/db/charset/DatabaseCharsetCheckerTest.java @@ -21,9 +21,6 @@ import java.sql.Connection; import java.sql.SQLException; -import java.util.Set; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -38,24 +35,21 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anySet; -import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.AUTO_REPAIR_COLLATION; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; public class DatabaseCharsetCheckerTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - Database db = mock(Database.class, Mockito.RETURNS_MOCKS); - CharsetHandler handler = mock(CharsetHandler.class); - DatabaseCharsetChecker underTest = spy(new DatabaseCharsetChecker(db)); + private Database db = mock(Database.class, Mockito.RETURNS_MOCKS); + private CharsetHandler handler = mock(CharsetHandler.class); + private DatabaseCharsetChecker underTest = spy(new DatabaseCharsetChecker(db)); @Test public void executes_handler() throws Exception { @@ -63,17 +57,8 @@ public void executes_handler() throws Exception { when(underTest.getHandler(dialect)).thenReturn(handler); when(db.getDialect()).thenReturn(dialect); - underTest.check(ENFORCE_UTF8); - verify(handler).handle(any(Connection.class), argThat(new TypeSafeMatcher>() { - @Override - protected boolean matchesSafely(Set flags) { - return flags.contains(ENFORCE_UTF8) && flags.size() == 1; - } - - @Override - public void describeTo(Description description) { - } - })); + underTest.check(DatabaseCharsetChecker.State.UPGRADE); + verify(handler).handle(any(Connection.class), eq(DatabaseCharsetChecker.State.UPGRADE)); } @Test @@ -81,11 +66,11 @@ public void throws_ISE_if_handler_fails() throws Exception { Oracle dialect = new Oracle(); when(underTest.getHandler(dialect)).thenReturn(handler); when(db.getDialect()).thenReturn(dialect); - doThrow(new SQLException("failure")).when(handler).handle(any(Connection.class), anySet()); + doThrow(new SQLException("failure")).when(handler).handle(any(Connection.class), any(DatabaseCharsetChecker.State.class)); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("failure"); - underTest.check(AUTO_REPAIR_COLLATION); + underTest.check(DatabaseCharsetChecker.State.UPGRADE); } @Test diff --git a/sonar-db/src/test/java/org/sonar/db/charset/MssqlCharsetHandlerTest.java b/sonar-db/src/test/java/org/sonar/db/charset/MssqlCharsetHandlerTest.java index bd81b60afd20..13e9dd78c7e8 100644 --- a/sonar-db/src/test/java/org/sonar/db/charset/MssqlCharsetHandlerTest.java +++ b/sonar-db/src/test/java/org/sonar/db/charset/MssqlCharsetHandlerTest.java @@ -25,26 +25,23 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; +import java.util.stream.Stream; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.sonar.api.utils.MessageException; -import static com.google.common.collect.Sets.immutableEnumSet; import static java.lang.String.format; import static java.util.Arrays.asList; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.AUTO_REPAIR_COLLATION; @RunWith(DataProviderRunner.class) public class MssqlCharsetHandlerTest { @@ -57,80 +54,98 @@ public class MssqlCharsetHandlerTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - SqlExecutor selectExecutor = mock(SqlExecutor.class); - MssqlCharsetHandler underTest = new MssqlCharsetHandler(selectExecutor); + private SqlExecutor sqlExecutor = mock(SqlExecutor.class); + private MssqlMetadataReader metadata = mock(MssqlMetadataReader.class); + private MssqlCharsetHandler underTest = new MssqlCharsetHandler(sqlExecutor, metadata); + private Connection connection = mock(Connection.class); @Test - public void do_not_fail_if_charsets_of_all_columns_are_CS_AS() throws Exception { - answerColumns(asList( - new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "Latin1_General", "Latin1_General_CS_AS", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CS_AS", "varchar", 10, false))); + public void fresh_install_verifies_that_default_collation_is_CS_AS() throws SQLException { + answerDefaultCollation("Latin1_General_CS_AS"); + + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); - underTest.handle(mock(Connection.class), Collections.emptySet()); + verify(metadata).getDefaultCollation(connection); } @Test - public void fail_if_a_column_is_case_insensitive_and_repair_is_disabled() throws Exception { - answerColumns(asList( - new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "Latin1_General", "Latin1_General_CS_AS", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "varchar", 10, false))); + public void fresh_install_fails_if_default_collation_is_not_CS_AS() throws SQLException { + answerDefaultCollation("Latin1_General_CI_AI"); expectedException.expect(MessageException.class); - expectedException.expectMessage("Case-sensitive and accent-sensitive collation is required for database columns [projects.name]"); - Connection connection = mock(Connection.class); - underTest.handle(connection, Collections.emptySet()); + expectedException.expectMessage("Database collation must be case-sensitive and accent-sensitive. It is Latin1_General_CI_AI but should be Latin1_General_CS_AS."); + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); + } + + @Test + public void upgrade_fails_if_default_collation_is_not_CS_AS() throws SQLException { + answerDefaultCollation("Latin1_General_CI_AI"); - verify(selectExecutor, never()).executeUpdate(any(Connection.class), anyString()); + expectedException.expect(MessageException.class); + expectedException.expectMessage("Database collation must be case-sensitive and accent-sensitive. It is Latin1_General_CI_AI but should be Latin1_General_CS_AS."); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); } @Test - public void repair_case_insensitive_column_without_index() throws Exception { - answerColumns(asList( + public void upgrade_checks_that_columns_are_CS_AS() throws SQLException { + answerDefaultCollation("Latin1_General_CS_AS"); + answerColumnDefs( new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "Latin1_General", "Latin1_General_CS_AS", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "varchar", 10, false))); + new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CS_AS", "varchar", 10, false)); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + // do not fail + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); + } - verify(selectExecutor).executeUpdate(connection, "ALTER TABLE projects ALTER COLUMN name varchar(10) COLLATE Latin1_General_CS_AS NOT NULL"); + @Test + public void upgrade_repairs_CI_AI_columns() throws SQLException { + answerDefaultCollation("Latin1_General_CS_AS"); + answerColumnDefs( + new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "Latin1_General", "Latin1_General_CS_AS", "varchar", 10, false), + new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "varchar", 10, false)); + + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); + + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE projects ALTER COLUMN name varchar(10) COLLATE Latin1_General_CS_AS NOT NULL"); } @Test - public void repair_case_insensitive_column_with_indices() throws Exception { - answerColumns(asList( + public void upgrade_repairs_indexed_CI_AI_columns() throws SQLException { + answerDefaultCollation("Latin1_General_CS_AS"); + answerColumnDefs( new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "Latin1_General", "Latin1_General_CS_AS", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "varchar", 10, false))); - answerIndices(asList( + new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "varchar", 10, false)); + answerIndices( new MssqlCharsetHandler.ColumnIndex("projects_name", false, "name"), // This index is on two columns. Note that it does not make sense for table "projects" ! - new MssqlCharsetHandler.ColumnIndex("projects_login_and_name", true, "login,name"))); + new MssqlCharsetHandler.ColumnIndex("projects_login_and_name", true, "login,name")); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - verify(selectExecutor).executeUpdate(connection, "DROP INDEX projects.projects_name"); - verify(selectExecutor).executeUpdate(connection, "DROP INDEX projects.projects_login_and_name"); - verify(selectExecutor).executeUpdate(connection, "ALTER TABLE projects ALTER COLUMN name varchar(10) COLLATE Latin1_General_CS_AS NOT NULL"); - verify(selectExecutor).executeUpdate(connection, "CREATE INDEX projects_name ON projects (name)"); - verify(selectExecutor).executeUpdate(connection, "CREATE UNIQUE INDEX projects_login_and_name ON projects (login,name)"); + verify(sqlExecutor).executeDdl(connection, "DROP INDEX projects.projects_name"); + verify(sqlExecutor).executeDdl(connection, "DROP INDEX projects.projects_login_and_name"); + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE projects ALTER COLUMN name varchar(10) COLLATE Latin1_General_CS_AS NOT NULL"); + verify(sqlExecutor).executeDdl(connection, "CREATE INDEX projects_name ON projects (name)"); + verify(sqlExecutor).executeDdl(connection, "CREATE UNIQUE INDEX projects_login_and_name ON projects (login,name)"); } @Test @UseDataProvider("combinationsOfCsAsAndSuffix") - public void repair_case_insensitive_accent_insensitive_combinations_with_or_without_suffix(String collation, String expectedCollation) throws Exception { - answerColumns(Collections.singletonList(new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "Latin1_General", collation, "varchar", 10, false))); + public void repair_case_insensitive_accent_insensitive_combinations_with_or_without_suffix(String collation, String expectedCollation) + throws Exception { + answerDefaultCollation("Latin1_General_CS_AS"); + answerColumnDefs(new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "Latin1_General", collation, "varchar", 10, false)); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - verify(selectExecutor).executeUpdate(connection, "ALTER TABLE issues ALTER COLUMN kee varchar(10) COLLATE " + expectedCollation + " NOT NULL"); + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE issues ALTER COLUMN kee varchar(10) COLLATE " + expectedCollation + " NOT NULL"); } @DataProvider public static Object[][] combinationsOfCsAsAndSuffix() { List res = new ArrayList<>(); - for (String sensitivity : Arrays.asList("CI_AI", "CI_AS", "CS_AI")) { - for (String suffix : Arrays.asList("", "_KS_WS")) { + for (String sensitivity : asList("CI_AI", "CI_AS", "CS_AI")) { + for (String suffix : asList("", "_KS_WS")) { res.add(new String[] { format("Latin1_General_%s%s", sensitivity, suffix), format("Latin1_General_CS_AS%s", suffix) @@ -142,41 +157,40 @@ public static Object[][] combinationsOfCsAsAndSuffix() { @Test public void support_the_max_size_of_varchar_column() throws Exception { + answerDefaultCollation("Latin1_General_CS_AS"); // returned size is -1 - answerColumns(asList(new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "nvarchar", -1, false))); - answerIndices(Collections.emptyList()); + answerColumnDefs(new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "nvarchar", -1, false)); + answerIndices(); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - verify(selectExecutor).executeUpdate(connection, "ALTER TABLE projects ALTER COLUMN name nvarchar(max) COLLATE Latin1_General_CS_AS NOT NULL"); + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE projects ALTER COLUMN name nvarchar(max) COLLATE Latin1_General_CS_AS NOT NULL"); } @Test public void do_not_repair_system_tables_of_sql_azure() throws Exception { - answerColumns(asList(new ColumnDef("sys.sysusers", COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "varchar", 10, false))); + answerDefaultCollation("Latin1_General_CS_AS"); + answerColumnDefs(new ColumnDef("sys.sysusers", COLUMN_NAME, "Latin1_General", "Latin1_General_CI_AI", "varchar", 10, false)); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - verify(selectExecutor, never()).executeUpdate(any(Connection.class), anyString()); + verify(sqlExecutor, never()).executeDdl(any(Connection.class), anyString()); } @Test @UseDataProvider("combinationOfBinAndSuffix") public void do_not_repair_if_collation_contains_BIN(String collation) throws Exception { - answerColumns(asList(new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", collation, "varchar", 10, false))); + answerDefaultCollation("Latin1_General_CS_AS"); + answerColumnDefs(new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", collation, "varchar", 10, false)); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - verify(selectExecutor, never()).executeUpdate(any(Connection.class), anyString()); + verify(sqlExecutor, never()).executeDdl(any(Connection.class), anyString()); } @DataProvider public static Object[][] combinationOfBinAndSuffix() { - return Arrays.asList("", "_KS_WS") - .stream() + return Stream.of("", "_KS_WS") .map(suffix -> new String[] {format("Latin1_General_BIN%s", suffix)}) .toArray(Object[][]::new); } @@ -184,27 +198,43 @@ public static Object[][] combinationOfBinAndSuffix() { @Test @UseDataProvider("combinationOfBin2AndSuffix") public void do_not_repair_if_collation_contains_BIN2(String collation) throws Exception { - answerColumns(asList(new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", collation, "varchar", 10, false))); + answerDefaultCollation("Latin1_General_CS_AS"); + answerColumnDefs(new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", collation, "varchar", 10, false)); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - verify(selectExecutor, never()).executeUpdate(any(Connection.class), anyString()); + verify(sqlExecutor, never()).executeDdl(any(Connection.class), anyString()); } @DataProvider public static Object[][] combinationOfBin2AndSuffix() { - return Arrays.asList("", "_KS_WS") - .stream() + return Stream.of("", "_KS_WS") .map(suffix -> new String[] {format("Latin1_General_BIN2%s", suffix)}) .toArray(Object[][]::new); } - private void answerColumns(List columnDefs) throws SQLException { - when(selectExecutor.executeSelect(any(Connection.class), anyString(), eq(ColumnDef.ColumnDefRowConverter.INSTANCE))).thenReturn(columnDefs); + /** + * SONAR-7988 + */ + @Test + public void fix_Latin1_CS_AS_columns_created_in_5_x() throws SQLException { + answerDefaultCollation("SQL_Latin1_General_CP1_CS_AS"); + answerColumnDefs(new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "Latin1_General", "Latin1_General_CS_AS", "nvarchar", 10, false)); + + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); + + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE projects ALTER COLUMN name nvarchar(10) COLLATE SQL_Latin1_General_CP1_CS_AS NOT NULL"); + } + + private void answerColumnDefs(ColumnDef... columnDefs) throws SQLException { + when(metadata.getColumnDefs(connection)).thenReturn(asList(columnDefs)); + } + + private void answerDefaultCollation(String defaultCollation) throws SQLException { + when(metadata.getDefaultCollation(connection)).thenReturn(defaultCollation); } - private void answerIndices(List indices) throws SQLException { - when(selectExecutor.executeSelect(any(Connection.class), anyString(), eq(MssqlCharsetHandler.ColumnIndexConverter.INSTANCE))).thenReturn(indices); + private void answerIndices(MssqlCharsetHandler.ColumnIndex... indices) throws SQLException { + when(metadata.getColumnIndices(same(connection), any(ColumnDef.class))).thenReturn(asList(indices)); } } diff --git a/sonar-db/src/test/java/org/sonar/db/charset/MssqlMetadataReaderTest.java b/sonar-db/src/test/java/org/sonar/db/charset/MssqlMetadataReaderTest.java new file mode 100644 index 000000000000..3901c201f684 --- /dev/null +++ b/sonar-db/src/test/java/org/sonar/db/charset/MssqlMetadataReaderTest.java @@ -0,0 +1,51 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.charset; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MssqlMetadataReaderTest { + + private SqlExecutor sqlExecutor = mock(SqlExecutor.class); + private Connection connection = mock(Connection.class); + private MssqlMetadataReader underTest = new MssqlMetadataReader(sqlExecutor); + + @Test + public void test_getDefaultCollation() throws SQLException { + answerSelect(Arrays.asList(new String[] {"Latin1_General_CS_AS"})); + + assertThat(underTest.getDefaultCollation(connection)).isEqualTo("Latin1_General_CS_AS"); + } + + private void answerSelect(List firstRequest) throws SQLException { + when(sqlExecutor.select(same(connection), anyString(), any(SqlExecutor.StringsConverter.class))).thenReturn(firstRequest); + } +} diff --git a/sonar-db/src/test/java/org/sonar/db/charset/MysqlCharsetHandlerTest.java b/sonar-db/src/test/java/org/sonar/db/charset/MysqlCharsetHandlerTest.java index d050fbedf3cf..8fc85450221d 100644 --- a/sonar-db/src/test/java/org/sonar/db/charset/MysqlCharsetHandlerTest.java +++ b/sonar-db/src/test/java/org/sonar/db/charset/MysqlCharsetHandlerTest.java @@ -21,23 +21,18 @@ import java.sql.Connection; import java.sql.SQLException; -import java.util.List; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.sonar.api.utils.MessageException; -import static com.google.common.collect.Sets.immutableEnumSet; import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.AUTO_REPAIR_COLLATION; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; public class MysqlCharsetHandlerTest { @@ -49,71 +44,56 @@ public class MysqlCharsetHandlerTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - SqlExecutor selectExecutor = mock(SqlExecutor.class); - MysqlCharsetHandler underTest = new MysqlCharsetHandler(selectExecutor); + private SqlExecutor sqlExecutor = mock(SqlExecutor.class); + private Connection connection = mock(Connection.class); + private MysqlCharsetHandler underTest = new MysqlCharsetHandler(sqlExecutor); @Test - public void do_not_fail_if_charsets_of_all_columns_are_utf8_and_case_sensitive() throws Exception { - answerColumnDef(asList( + public void upgrade_verifies_that_columns_are_utf8_and_case_sensitive() throws Exception { + answerColumnDef( new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "utf8", "utf8_bin", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "utf8", "utf8_bin", "varchar", 10, false))); + new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "utf8", "utf8_bin", "varchar", 10, false)); // all columns are utf8 - underTest.handle(mock(Connection.class), immutableEnumSet(ENFORCE_UTF8)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); } @Test - public void fail_if_charsets_of_a_column_is_utf8_but_case_insensitive() throws Exception { - answerColumnDef(asList( - new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "utf8", "utf8_bin", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "utf8", "utf8_general_ci", "varchar", 10, false))); - - expectedException.expect(MessageException.class); - expectedException.expectMessage("UTF8 case-sensitive collation is required for database columns [projects.name]"); - - underTest.handle(mock(Connection.class), immutableEnumSet(ENFORCE_UTF8)); + public void fresh_install_does_not_verify_anything() throws Exception { + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); + verifyZeroInteractions(sqlExecutor); } @Test - public void fail_if_not_utf8() throws Exception { - answerColumnDef(asList( - new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "utf8", "utf8_bin", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_KEE, "latin1", "latin1_german1_ci", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "latin1", "latin1_swedish_ci", "varchar", 20, false))); - - expectedException.expect(MessageException.class); - expectedException.expectMessage("UTF8 case-sensitive collation is required for database columns [projects.kee, projects.name]"); - underTest.handle(mock(Connection.class), immutableEnumSet(ENFORCE_UTF8)); + public void regular_startup_does_not_verify_anything() throws Exception { + underTest.handle(connection, DatabaseCharsetChecker.State.STARTUP); + verifyZeroInteractions(sqlExecutor); } @Test public void repair_case_insensitive_column() throws Exception { - answerColumnDef(asList( - new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "utf8", "utf8_bin", "varchar", 10, false), - new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "latin1", "latin1_swedish_ci", "varchar", 10, false))); + answerColumnDef( + new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "big5_chinese", "big5_chinese_ci", "varchar", 10, false), + new ColumnDef(TABLE_PROJECTS, COLUMN_NAME, "latin1", "latin1_swedish_ci", "varchar", 10, false)); - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - verify(selectExecutor).executeUpdate(connection, "ALTER TABLE projects MODIFY name varchar(10) CHARACTER SET 'latin1' COLLATE 'latin1_bin' NOT NULL"); + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE issues MODIFY kee varchar(10) CHARACTER SET 'big5_chinese' COLLATE 'big5_bin' NOT NULL"); + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE projects MODIFY name varchar(10) CHARACTER SET 'latin1' COLLATE 'latin1_bin' NOT NULL"); } @Test public void size_should_be_ignored_on_longtext_column() throws Exception { - answerColumnDef(asList(new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "latin1", "latin1_german1_ci", "longtext", 4_294_967_295L, false))); - - Connection connection = mock(Connection.class); - underTest.handle(connection, immutableEnumSet(AUTO_REPAIR_COLLATION)); + answerColumnDef( + new ColumnDef(TABLE_ISSUES, COLUMN_KEE, "latin1", "latin1_german1_ci", "longtext", 4_294_967_295L, false)); - verify(selectExecutor).executeUpdate(connection, "ALTER TABLE " + TABLE_ISSUES + " MODIFY " + COLUMN_KEE + " longtext CHARACTER SET 'latin1' COLLATE 'latin1_bin' NOT NULL"); - } + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - @Test - public void tests_toCaseSensitive() { - assertThat(MysqlCharsetHandler.toCaseSensitive("big5_chinese_ci")).isEqualTo("big5_bin"); + verify(sqlExecutor).executeDdl(connection, "ALTER TABLE " + TABLE_ISSUES + " MODIFY " + COLUMN_KEE + " longtext CHARACTER SET 'latin1' COLLATE 'latin1_bin' NOT NULL"); } - private void answerColumnDef(List columnDefs) throws SQLException { - when(selectExecutor.executeSelect(any(Connection.class), anyString(), eq(ColumnDef.ColumnDefRowConverter.INSTANCE))).thenReturn(columnDefs); + private void answerColumnDef(ColumnDef... columnDefs) throws SQLException { + when(sqlExecutor.select(any(Connection.class), anyString(), eq(ColumnDef.ColumnDefRowConverter.INSTANCE))) + .thenReturn(asList(columnDefs)); } } diff --git a/sonar-db/src/test/java/org/sonar/db/charset/OracleCharsetHandlerTest.java b/sonar-db/src/test/java/org/sonar/db/charset/OracleCharsetHandlerTest.java index 76be06e7f21f..f0474c2475b6 100644 --- a/sonar-db/src/test/java/org/sonar/db/charset/OracleCharsetHandlerTest.java +++ b/sonar-db/src/test/java/org/sonar/db/charset/OracleCharsetHandlerTest.java @@ -23,84 +23,87 @@ import java.sql.SQLException; import java.util.Collections; import java.util.List; -import java.util.Set; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.sonar.api.utils.MessageException; -import org.sonar.db.charset.DatabaseCharsetChecker.Flag; -import static com.google.common.collect.Sets.immutableEnumSet; import static java.util.Collections.singletonList; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; public class OracleCharsetHandlerTest { - private static final Set ENFORCE_UTF8_FLAGS = immutableEnumSet(ENFORCE_UTF8); - @Rule public ExpectedException expectedException = ExpectedException.none(); - SqlExecutor selectExecutor = mock(SqlExecutor.class); - OracleCharsetHandler underTest = new OracleCharsetHandler(selectExecutor); + private SqlExecutor sqlExecutor = mock(SqlExecutor.class); + private Connection connection = mock(Connection.class); + private OracleCharsetHandler underTest = new OracleCharsetHandler(sqlExecutor); @Test - public void checks_utf8() throws Exception { - answerSql( - singletonList(new String[] {"UTF8"}), singletonList(new String[] {"BINARY"})); + public void fresh_install_verifies_utf8_charset() throws Exception { + answerSql(singletonList(new String[] {"UTF8"}), singletonList(new String[] {"BINARY"})); + + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); + } + + @Test + public void upgrade_does_not_verify_utf8_charset() throws Exception { + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); - underTest.handle(mock(Connection.class), ENFORCE_UTF8_FLAGS); + verifyZeroInteractions(sqlExecutor); } @Test - public void supports_al32utf8() throws Exception { + public void fresh_install_supports_al32utf8() throws Exception { answerSql( singletonList(new String[] {"AL32UTF8"}), singletonList(new String[] {"BINARY"})); - underTest.handle(mock(Connection.class), ENFORCE_UTF8_FLAGS); + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); } @Test - public void fails_if_charset_is_not_utf8() throws Exception { + public void fresh_install_fails_if_charset_is_not_utf8() throws Exception { answerSql( singletonList(new String[] {"LATIN"}), singletonList(new String[] {"BINARY"})); expectedException.expect(MessageException.class); expectedException.expectMessage("Oracle must be have UTF8 charset and BINARY sort. NLS_CHARACTERSET is LATIN and NLS_SORT is BINARY."); - underTest.handle(mock(Connection.class), ENFORCE_UTF8_FLAGS); + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); } @Test - public void fails_if_not_case_sensitive() throws Exception { + public void fresh_install_fails_if_not_case_sensitive() throws Exception { answerSql( singletonList(new String[] {"UTF8"}), singletonList(new String[] {"LINGUISTIC"})); expectedException.expect(MessageException.class); expectedException.expectMessage("Oracle must be have UTF8 charset and BINARY sort. NLS_CHARACTERSET is UTF8 and NLS_SORT is LINGUISTIC."); - underTest.handle(mock(Connection.class), ENFORCE_UTF8_FLAGS); + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); } @Test public void fails_if_can_not_get_charset() throws Exception { - answerSql(Collections.emptyList(), Collections.emptyList()); + answerSql(Collections.emptyList(), Collections.emptyList()); expectedException.expect(MessageException.class); - underTest.handle(mock(Connection.class), ENFORCE_UTF8_FLAGS); + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); } @Test - public void does_nothing_if_utf8_must_not_verified() throws Exception { - underTest.handle(mock(Connection.class), Collections.emptySet()); + public void does_nothing_if_regular_startup() throws Exception { + underTest.handle(connection, DatabaseCharsetChecker.State.STARTUP); + verifyZeroInteractions(sqlExecutor); } private void answerSql(List firstRequest, List... otherRequests) throws SQLException { - when(selectExecutor.executeSelect(any(Connection.class), anyString(), any(SqlExecutor.StringsConverter.class))).thenReturn(firstRequest, otherRequests); + when(sqlExecutor.select(any(Connection.class), anyString(), any(SqlExecutor.StringsConverter.class))).thenReturn(firstRequest, otherRequests); } } diff --git a/sonar-db/src/test/java/org/sonar/db/charset/PostgresCharsetHandlerTest.java b/sonar-db/src/test/java/org/sonar/db/charset/PostgresCharsetHandlerTest.java index d348e31117e4..69f0fb767208 100644 --- a/sonar-db/src/test/java/org/sonar/db/charset/PostgresCharsetHandlerTest.java +++ b/sonar-db/src/test/java/org/sonar/db/charset/PostgresCharsetHandlerTest.java @@ -22,21 +22,20 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.Arrays; -import java.util.Collections; import java.util.List; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.sonar.api.utils.MessageException; -import org.sonar.db.charset.DatabaseCharsetChecker.Flag; -import static com.google.common.collect.Sets.immutableEnumSet; import static java.util.Arrays.asList; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.same; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; -import static org.sonar.db.charset.DatabaseCharsetChecker.Flag.ENFORCE_UTF8; public class PostgresCharsetHandlerTest { @@ -48,69 +47,88 @@ public class PostgresCharsetHandlerTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - SqlExecutor selectExecutor = mock(SqlExecutor.class); - PostgresCharsetHandler underTest = new PostgresCharsetHandler(selectExecutor); + private SqlExecutor sqlExecutor = mock(SqlExecutor.class); + private Connection connection = mock(Connection.class); + private PostgresMetadataReader metadata = mock(PostgresMetadataReader.class); + private PostgresCharsetHandler underTest = new PostgresCharsetHandler(sqlExecutor, metadata); @Test - public void checks_that_column_is_utf8() throws Exception { - answerSql(asList( + public void fresh_install_verifies_that_default_charset_is_utf8() throws SQLException { + answerDefaultCharset("utf8"); + + underTest.handle(connection, DatabaseCharsetChecker.State.FRESH_INSTALL); + // no errors, charset has been verified + verify(metadata).getDefaultCharset(same(connection)); + verifyZeroInteractions(sqlExecutor); + } + + @Test + public void upgrade_verifies_that_default_charset_and_columns_are_utf8() throws Exception { + answerDefaultCharset("utf8"); + answerColumns(asList( new String[] {TABLE_ISSUES, COLUMN_KEE, "utf8"}, new String[] {TABLE_PROJECTS, COLUMN_NAME, "utf8"})); - underTest.handle(mock(Connection.class), immutableEnumSet(ENFORCE_UTF8)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); + // no errors, charsets have been verified + verify(metadata).getDefaultCharset(same(connection)); } @Test - public void checks_that_db_is_utf8_if_column_collation_is_not_defined() throws Exception { - answerSql( - // first request to get columns - asList( - new String[] {TABLE_ISSUES, COLUMN_KEE, "utf8"}, - new String[] {TABLE_PROJECTS, COLUMN_NAME, "" /* unset -> uses db collation */}), + public void regular_startup_verifies_that_default_charset_and_columns_are_utf8() throws Exception { + answerDefaultCharset("utf8"); + answerColumns(asList( + new String[] {TABLE_ISSUES, COLUMN_KEE, "utf8"}, + new String[] {TABLE_PROJECTS, COLUMN_NAME, "utf8"})); - // second request to get db collation - Arrays.asList(new String[] {"utf8"})); + underTest.handle(connection, DatabaseCharsetChecker.State.STARTUP); + // no errors, charsets have been verified + verify(metadata).getDefaultCharset(same(connection)); + } + + @Test + public void column_charset_can_be_empty() throws Exception { + answerDefaultCharset("utf8"); + answerColumns(asList( + new String[] {TABLE_ISSUES, COLUMN_KEE, "utf8"}, + new String[] {TABLE_PROJECTS, COLUMN_NAME, "" /* unset -> uses db collation */})); // no error - underTest.handle(mock(Connection.class), immutableEnumSet(ENFORCE_UTF8)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); } @Test - public void fails_if_non_utf8_column() throws Exception { - answerSql(asList( + public void upgrade_fails_if_non_utf8_column() throws Exception { + // default charset is ok but two columns are not + answerDefaultCharset("utf8"); + answerColumns(asList( new String[] {TABLE_ISSUES, COLUMN_KEE, "utf8"}, new String[] {TABLE_PROJECTS, COLUMN_KEE, "latin"}, new String[] {TABLE_PROJECTS, COLUMN_NAME, "latin"})); expectedException.expect(MessageException.class); - expectedException.expectMessage("Database columns [projects.kee, projects.name] must support UTF8 collation."); + expectedException.expectMessage("Database columns [projects.kee, projects.name] must have UTF8 charset."); - underTest.handle(mock(Connection.class), immutableEnumSet(ENFORCE_UTF8)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); } @Test - public void fails_if_non_utf8_db() throws Exception { - answerSql( - // first request to get columns - asList( - new String[] {TABLE_ISSUES, COLUMN_KEE, "utf8"}, - new String[] {TABLE_PROJECTS, COLUMN_NAME, "" /* unset -> uses db collation */}), - - // second request to get db collation - Arrays.asList(new String[] {"latin"})); + public void upgrade_fails_if_default_charset_is_not_utf8() throws Exception { + answerDefaultCharset("latin"); + answerColumns( + Arrays.asList(new String[] {TABLE_ISSUES, COLUMN_KEE, "utf8"})); expectedException.expect(MessageException.class); - expectedException.expectMessage("Database collation is latin. It must support UTF8."); + expectedException.expectMessage("Database charset is latin. It must support UTF8."); - underTest.handle(mock(Connection.class), immutableEnumSet(ENFORCE_UTF8)); + underTest.handle(connection, DatabaseCharsetChecker.State.UPGRADE); } - @Test - public void does_nothing_if_utf8_must_not_verified() throws Exception { - underTest.handle(mock(Connection.class), Collections.emptySet()); + private void answerDefaultCharset(String defaultCollation) throws SQLException { + when(metadata.getDefaultCharset(same(connection))).thenReturn(defaultCollation); } - private void answerSql(List firstRequest, List... otherRequests) throws SQLException { - when(selectExecutor.executeSelect(any(Connection.class), anyString(), any(SqlExecutor.StringsConverter.class))).thenReturn(firstRequest, otherRequests); + private void answerColumns(List firstRequest) throws SQLException { + when(sqlExecutor.select(same(connection), anyString(), any(SqlExecutor.StringsConverter.class))).thenReturn(firstRequest); } } diff --git a/sonar-db/src/test/java/org/sonar/db/charset/PostgresMetadataReaderTest.java b/sonar-db/src/test/java/org/sonar/db/charset/PostgresMetadataReaderTest.java new file mode 100644 index 000000000000..edcb4a823e98 --- /dev/null +++ b/sonar-db/src/test/java/org/sonar/db/charset/PostgresMetadataReaderTest.java @@ -0,0 +1,52 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.charset; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PostgresMetadataReaderTest { + + private SqlExecutor sqlExecutor = mock(SqlExecutor.class); + private Connection connection = mock(Connection.class); + private PostgresMetadataReader underTest = new PostgresMetadataReader(sqlExecutor); + + @Test + public void test_getDefaultCharset() throws SQLException { + answerSelect(Arrays.asList(new String[] {"latin"})); + + assertThat(underTest.getDefaultCharset(connection)).isEqualTo("latin"); + } + + private void answerSelect(List firstRequest) throws SQLException { + when(sqlExecutor.select(same(connection), anyString(), any(SqlExecutor.StringsConverter.class))).thenReturn(firstRequest); + } + +} diff --git a/sonar-db/src/test/java/org/sonar/db/charset/SelectExecutorTest.java b/sonar-db/src/test/java/org/sonar/db/charset/SelectExecutorTest.java index 5c36b5f6cb85..bdcee828c2a0 100644 --- a/sonar-db/src/test/java/org/sonar/db/charset/SelectExecutorTest.java +++ b/sonar-db/src/test/java/org/sonar/db/charset/SelectExecutorTest.java @@ -45,7 +45,7 @@ public void testExecuteQuery() throws Exception { session.commit(); try (Connection connection = dbTester.openConnection()) { - List rows = underTest.executeSelect(connection, "select login, name from users order by login", new SqlExecutor.StringsConverter(2)); + List rows = underTest.select(connection, "select login, name from users order by login", new SqlExecutor.StringsConverter(2)); assertThat(rows).hasSize(2); assertThat(rows.get(0)[0]).isEqualTo("her"); assertThat(rows.get(0)[1]).isEqualTo("Her"); diff --git a/sonar-db/src/test/java/org/sonar/db/charset/SqlExecutorTest.java b/sonar-db/src/test/java/org/sonar/db/charset/SqlExecutorTest.java index 4b4da278cb21..f107e2bb5c39 100644 --- a/sonar-db/src/test/java/org/sonar/db/charset/SqlExecutorTest.java +++ b/sonar-db/src/test/java/org/sonar/db/charset/SqlExecutorTest.java @@ -58,7 +58,7 @@ public void executeSelect_executes_PreparedStatement() throws Exception { dbTester.executeInsert(USERS_DB_TABLE, ImmutableMap.of(LOGIN_DB_COLUMN, "login2", NAME_DB_COLUMN, "name two")); try (Connection connection = dbTester.openConnection()) { - List users = underTest.executeSelect(connection, "select " + LOGIN_DB_COLUMN + ", " + NAME_DB_COLUMN + " from users order by id", new SqlExecutor.StringsConverter( + List users = underTest.select(connection, "select " + LOGIN_DB_COLUMN + ", " + NAME_DB_COLUMN + " from users order by id", new SqlExecutor.StringsConverter( 2)); assertThat(users).hasSize(2); assertThat(users.get(0)[0]).isEqualTo("login1"); @@ -73,7 +73,7 @@ public void executeUpdate_executes_PreparedStatement() throws Exception { dbTester.executeInsert(USERS_DB_TABLE, ImmutableMap.of(LOGIN_DB_COLUMN, "the_login", NAME_DB_COLUMN, "the name")); try (Connection connection = dbTester.openConnection()) { - underTest.executeUpdate(connection, "update users set " + NAME_DB_COLUMN + "='new name' where " + LOGIN_DB_COLUMN + "='the_login'"); + underTest.executeDdl(connection, "update users set " + NAME_DB_COLUMN + "='new name' where " + LOGIN_DB_COLUMN + "='the_login'"); } Map row = dbTester.selectFirst("select " + NAME_DB_COLUMN + " from users where " + LOGIN_DB_COLUMN + "='the_login'"); assertThat(row).isNotEmpty();