FINERACT-1398: Standardize charset and collation defaults

fineract-provider/build.gradle

@@ -185,7 +185,7 @@ tasks.register('createDB') {
     description = "Creates the MariaDB Database. Needs database name to be passed (like: -PdbName=someDBname)"
     doLast {
         def sql = Sql.newInstance('jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver')
-        sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4")
+        sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
     }
 }
 
@@ -217,7 +217,7 @@ tasks.register('createMySQLDB') {
     description = "Creates the MySQL Database. Needs database name to be passed (like: -PdbName=someDBname)"
     doLast {
         def sql = Sql.newInstance('jdbc:mysql://localhost:3306/', mysqlUser, mysqlPassword, 'com.mysql.cj.jdbc.Driver')
-        sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4")
+        sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
     }
 }
 

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java

@@ -277,7 +277,7 @@ public CommandProcessingResult createDatatable(final JsonCommand command) {
             sqlBuilder.append(constrainBuilder);
             sqlBuilder.append(")");
             if (databaseTypeResolver.isMySQL()) {
-                sqlBuilder.append(" ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4;");
+                sqlBuilder.append(" ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4 COLLATE=UTF8MB4_UNICODE_CI;");
             }
             log.debug("SQL:: {}", sqlBuilder);
 

fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml

@@ -31,4 +31,5 @@
      <include file="parts/0008_encrypt_existing_ro_tenant_passwords.xml" relativeToChangelogFile="true"/>
      <include file="parts/0009_set_and_encrypt_ro_if_not_exists.xml" relativeToChangelogFile="true"/>
      <include file="parts/0010_set_datetime_precision.xml" relativeToChangelogFile="true"/>
+     <include file="parts/0011_standardize_character_set_and_collation.xml" relativeToChangelogFile="true"/>
 </databaseChangeLog>

fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements. See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership. The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied. See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd">
+    <changeSet author="fineract" id="1" context="tenant_store_db">
+        <sql dbms="mysql,mariadb">ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;</sql>
+    </changeSet>
+</databaseChangeLog>

fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml

@@ -235,4 +235,5 @@
     <include file="parts/0214_trial_balance_summary_adding_originators.xml" relativeToChangelogFile="true" />
     <include file="parts/0215_transaction_summary_reports_add_buydown_fee_types.xml" relativeToChangelogFile="true" />
     <include file="parts/0216_add_unique_constraint_sms_campaign_name.xml" relativeToChangelogFile="true" />
+    <include file="parts/0217_standardize_character_set_and_collation.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>

fineract-provider/src/main/resources/db/changelog/tenant/parts/0217_standardize_character_set_and_collation.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements. See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership. The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied. See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd">
+    <changeSet author="fineract" id="1">
+        <sql dbms="mysql,mariadb">ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;</sql>
+    </changeSet>
+</databaseChangeLog>

fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java

@@ -24,21 +24,27 @@
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.fineract.infrastructure.codes.service.CodeReadPlatformService;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.api.JsonCommand;
 import org.apache.fineract.infrastructure.core.serialization.DatatableCommandFromApiJsonDeserializer;
 import org.apache.fineract.infrastructure.core.serialization.FromJsonHelper;
 import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator;
+import org.apache.fineract.infrastructure.core.service.database.DatabaseType;
 import org.apache.fineract.infrastructure.core.service.database.DatabaseTypeResolver;
 import org.apache.fineract.infrastructure.dataqueries.data.DataTableValidator;
+import org.apache.fineract.infrastructure.dataqueries.data.EntityTables;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
 import org.apache.fineract.portfolio.search.service.SearchUtil;
@@ -245,4 +251,51 @@ void testRegisterColumnCodeMappingUsesParameterizedQuery() {
         assertTrue(sql.contains("INSERT INTO x_table_column_code_mappings"), "SQL should insert into code mappings table");
         assertTrue(sql.contains("VALUES (?, ?)"), "SQL should use ? placeholders");
     }
+
+    @Test
+    void testCreateDatatableUsesUtf8mb4UnicodeCiForMySql() {
+        final JsonElement payload = JsonParser.parseString("""
+                {
+                  "datatableName": "dt_charset_test",
+                  "apptableName": "m_client",
+                  "entitySubType": "PERSON",
+                  "multiRow": false,
+                  "columns": [
+                    {
+                      "name": "itsAString",
+                      "type": "String",
+                      "mandatory": true,
+                      "length": 10
+                    }
+                  ]
+                }
+                """);
+
+        final JsonCommand command = mock(JsonCommand.class);
+        when(command.json()).thenReturn(payload.toString());
+        when(command.commandId()).thenReturn(1L);
+
+        when(databaseTypeResolver.isMySQL()).thenReturn(true);
+        when(databaseTypeResolver.databaseType()).thenReturn(DatabaseType.MYSQL);
+        when(configurationDomainService.isConstraintApproachEnabledForDatatables()).thenReturn(false);
+        when(sqlGenerator.currentSchema()).thenReturn("database()");
+        when(sqlGenerator.escape(anyString())).thenAnswer(invocation -> "`" + invocation.getArgument(0) + "`");
+        when(datatableUtil.resolveEntity("m_client")).thenReturn(EntityTables.CLIENT);
+        when(datatableUtil.getFKField(EntityTables.CLIENT)).thenReturn("client_id");
+        when(fromJsonHelper.parse(anyString())).thenReturn(payload);
+        when(fromJsonHelper.extractJsonArrayNamed(eq("columns"), eq(payload)))
+                .thenReturn(payload.getAsJsonObject().getAsJsonArray("columns"));
+        when(fromJsonHelper.extractStringNamed(eq("datatableName"), eq(payload))).thenReturn("dt_charset_test");
+        when(fromJsonHelper.extractStringNamed(eq("entitySubType"), eq(payload))).thenReturn("PERSON");
+        when(fromJsonHelper.extractStringNamed(eq("apptableName"), eq(payload))).thenReturn("m_client");
+        when(fromJsonHelper.extractBooleanNamed(eq("multiRow"), eq(payload))).thenReturn(false);
+        when(jdbcTemplate.queryForObject(anyString(), eq(String.class), eq("dt_charset_test"))).thenReturn("true");
+
+        underTest.createDatatable(command);
+
+        final ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
+        verify(jdbcTemplate).execute(sqlCaptor.capture());
+        assertTrue(sqlCaptor.getValue().contains("ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4 COLLATE=UTF8MB4_UNICODE_CI;"),
+                "MySQL table creation must include utf8mb4 charset and utf8mb4_unicode_ci collation");
+    }
 }