From b037011e39bf91226e488d05fe0e92b3fda85b45 Mon Sep 17 00:00:00 2001 From: airajena Date: Mon, 2 Mar 2026 23:21:28 +0530 Subject: [PATCH] FINERACT-1398: Standardize charset and collation defaults --- fineract-provider/build.gradle | 4 +- .../service/DatatableWriteServiceImpl.java | 2 +- .../tenant-store/changelog-tenant-store.xml | 1 + ...tandardize_character_set_and_collation.xml | 28 ++++++++++ .../db/changelog/tenant/changelog-tenant.xml | 1 + ...tandardize_character_set_and_collation.xml | 28 ++++++++++ .../DatatableWriteServiceImplTest.java | 53 +++++++++++++++++++ 7 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml create mode 100644 fineract-provider/src/main/resources/db/changelog/tenant/parts/0217_standardize_character_set_and_collation.xml diff --git a/fineract-provider/build.gradle b/fineract-provider/build.gradle index 4ab05300f10..6de9c06cab1 100644 --- a/fineract-provider/build.gradle +++ b/fineract-provider/build.gradle @@ -185,7 +185,7 @@ tasks.register('createDB') { description = "Creates the MariaDB Database. Needs database name to be passed (like: -PdbName=someDBname)" doLast { def sql = Sql.newInstance('jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver') - sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4") + sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci") } } @@ -217,7 +217,7 @@ tasks.register('createMySQLDB') { description = "Creates the MySQL Database. Needs database name to be passed (like: -PdbName=someDBname)" doLast { def sql = Sql.newInstance('jdbc:mysql://localhost:3306/', mysqlUser, mysqlPassword, 'com.mysql.cj.jdbc.Driver') - sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4") + sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci") } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java index 5c07474ea5e..3f81b76674e 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java @@ -277,7 +277,7 @@ public CommandProcessingResult createDatatable(final JsonCommand command) { sqlBuilder.append(constrainBuilder); sqlBuilder.append(")"); if (databaseTypeResolver.isMySQL()) { - sqlBuilder.append(" ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4;"); + sqlBuilder.append(" ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4 COLLATE=UTF8MB4_UNICODE_CI;"); } log.debug("SQL:: {}", sqlBuilder); diff --git a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml b/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml index de38521dae5..7d5a94e7835 100644 --- a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml +++ b/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml @@ -31,4 +31,5 @@ + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml b/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml new file mode 100644 index 00000000000..7825c416067 --- /dev/null +++ b/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml @@ -0,0 +1,28 @@ + + + + + ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml index faf3484d674..61830785cfb 100644 --- a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml +++ b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml @@ -235,4 +235,5 @@ + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/parts/0217_standardize_character_set_and_collation.xml b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0217_standardize_character_set_and_collation.xml new file mode 100644 index 00000000000..9b995bc8639 --- /dev/null +++ b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0217_standardize_character_set_and_collation.xml @@ -0,0 +1,28 @@ + + + + + ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + + diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java index 44be562808f..a1cd828842f 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java @@ -24,21 +24,27 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.fineract.infrastructure.codes.service.CodeReadPlatformService; import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; +import org.apache.fineract.infrastructure.core.api.JsonCommand; import org.apache.fineract.infrastructure.core.serialization.DatatableCommandFromApiJsonDeserializer; import org.apache.fineract.infrastructure.core.serialization.FromJsonHelper; import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator; +import org.apache.fineract.infrastructure.core.service.database.DatabaseType; import org.apache.fineract.infrastructure.core.service.database.DatabaseTypeResolver; import org.apache.fineract.infrastructure.dataqueries.data.DataTableValidator; +import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.portfolio.search.service.SearchUtil; @@ -245,4 +251,51 @@ void testRegisterColumnCodeMappingUsesParameterizedQuery() { assertTrue(sql.contains("INSERT INTO x_table_column_code_mappings"), "SQL should insert into code mappings table"); assertTrue(sql.contains("VALUES (?, ?)"), "SQL should use ? placeholders"); } + + @Test + void testCreateDatatableUsesUtf8mb4UnicodeCiForMySql() { + final JsonElement payload = JsonParser.parseString(""" + { + "datatableName": "dt_charset_test", + "apptableName": "m_client", + "entitySubType": "PERSON", + "multiRow": false, + "columns": [ + { + "name": "itsAString", + "type": "String", + "mandatory": true, + "length": 10 + } + ] + } + """); + + final JsonCommand command = mock(JsonCommand.class); + when(command.json()).thenReturn(payload.toString()); + when(command.commandId()).thenReturn(1L); + + when(databaseTypeResolver.isMySQL()).thenReturn(true); + when(databaseTypeResolver.databaseType()).thenReturn(DatabaseType.MYSQL); + when(configurationDomainService.isConstraintApproachEnabledForDatatables()).thenReturn(false); + when(sqlGenerator.currentSchema()).thenReturn("database()"); + when(sqlGenerator.escape(anyString())).thenAnswer(invocation -> "`" + invocation.getArgument(0) + "`"); + when(datatableUtil.resolveEntity("m_client")).thenReturn(EntityTables.CLIENT); + when(datatableUtil.getFKField(EntityTables.CLIENT)).thenReturn("client_id"); + when(fromJsonHelper.parse(anyString())).thenReturn(payload); + when(fromJsonHelper.extractJsonArrayNamed(eq("columns"), eq(payload))) + .thenReturn(payload.getAsJsonObject().getAsJsonArray("columns")); + when(fromJsonHelper.extractStringNamed(eq("datatableName"), eq(payload))).thenReturn("dt_charset_test"); + when(fromJsonHelper.extractStringNamed(eq("entitySubType"), eq(payload))).thenReturn("PERSON"); + when(fromJsonHelper.extractStringNamed(eq("apptableName"), eq(payload))).thenReturn("m_client"); + when(fromJsonHelper.extractBooleanNamed(eq("multiRow"), eq(payload))).thenReturn(false); + when(jdbcTemplate.queryForObject(anyString(), eq(String.class), eq("dt_charset_test"))).thenReturn("true"); + + underTest.createDatatable(command); + + final ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(jdbcTemplate).execute(sqlCaptor.capture()); + assertTrue(sqlCaptor.getValue().contains("ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4 COLLATE=UTF8MB4_UNICODE_CI;"), + "MySQL table creation must include utf8mb4 charset and utf8mb4_unicode_ci collation"); + } }