Skip to content

FINERACT-1693: Rework datatable entry handling + added additional tests #2511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
galovics merged 1 commit into from
Aug 15, 2022
Merged

FINERACT-1693: Rework datatable entry handling + added additional tests #2511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions ...n/java/org/apache/fineract/infrastructure/dataqueries/service/GenericDataServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -82,10 +81,18 @@ public GenericResultsetData fillGenericResultSet(final String sql) {
final List<Object> columnValues = new ArrayList<>();
for (int i = 0; i < rsmd.getColumnCount(); i++) {
final String columnName = rsmd.getColumnName(i + 1);
final String columnValue = rs.getString(columnName);
columnValues.add(columnValue);
final String colType = columnHeaders.get(i).getColumnType();
if ("DATE".equalsIgnoreCase(colType)) {
java.sql.Date tmpDate = (java.sql.Date) rs.getObject(columnName);
columnValues.add(tmpDate != null ? tmpDate.toLocalDate() : null);
} else if ("TIMESTAMP WITHOUT TIME ZONE".equalsIgnoreCase(colType) // PostgreSQL
|| "DATETIME".equalsIgnoreCase(colType) || "TIMESTAMP".equalsIgnoreCase(colType)) {
Timestamp tmpDate = (Timestamp) rs.getObject(columnName);
columnValues.add(tmpDate != null ? tmpDate.toLocalDateTime() : null);
} else {
columnValues.add(rs.getObject(columnName));
}
}

final ResultsetRowData resultsetDataRow = ResultsetRowData.create(columnValues);
resultsetDataRows.add(resultsetDataRow);
}
Expand Down Expand Up @@ -167,19 +174,20 @@ public String generateJsonFromGenericResultsetData(final GenericResultsetData gr
}
currVal = row.get(j);
if (currVal != null && currColType != null) {
if (currColType.equals("DECIMAL") || currColType.equals("INTEGER")) {
if (currColType.equalsIgnoreCase("DECIMAL") || currColType.equalsIgnoreCase("INTEGER")
|| currColType.equalsIgnoreCase("CODELOOKUP")) {
writer.append(currVal);
} else {
if (currColType.equals("DATE")) {
final LocalDate localDate = new java.sql.Date(((Date) currVal).getTime()).toLocalDate();
if (currColType.equalsIgnoreCase("DATE")) {
final LocalDate localDate = (LocalDate) currVal;
writer.append(format("[%d,%d,%d]", localDate.getYear(), localDate.getMonthValue(), localDate.getDayOfMonth()));
} else if (currColType.equals("DATETIME")) {
final LocalDateTime ldt = currVal instanceof Timestamp ? ((Timestamp) currVal).toLocalDateTime()
: (LocalDateTime) currVal;
writer.append(format("[%d,%d,%d,%d,%d,%d,%d]", ldt.getYear(), ldt.getMonthValue(), ldt.getDayOfMonth(),
ldt.getHour(), ldt.getMinute(), ldt.getSecond(), ldt.getNano()));
} else if (currColType.equalsIgnoreCase("DATETIME")) {
final LocalDateTime localDateTime = (LocalDateTime) currVal;
writer.append(format("[%d,%d,%d,%d,%d,%d,%d]", localDateTime.getYear(), localDateTime.getMonthValue(),
localDateTime.getDayOfMonth(), localDateTime.getHour(), localDateTime.getMinute(),
localDateTime.getSecond(), localDateTime.getNano()));
} else {
writer.append(doubleQuote + replace((String) currVal, doubleQuote, slashDoubleQuote) + doubleQuote);
writer.append(doubleQuote + replace(String.valueOf(currVal), doubleQuote, slashDoubleQuote) + doubleQuote);
}
}
} else {
Expand Down
129 changes: 46 additions & 83 deletions ...g/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@
import com.google.gson.reflect.TypeToken;
import java.lang.reflect.Type;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
Expand Down Expand Up @@ -1231,8 +1231,6 @@ private CommandProcessingResult updateDatatableEntry(final String dataTableName,
LOG.info("Update sql: {}", sql);
if (StringUtils.isNotBlank(sql)) {
this.jdbcTemplate.update(sql);
changes.put("locale", dataParams.get("locale"));
changes.put("dateFormat", "yyyy-MM-dd");
} else {
LOG.info("No Changes");
}
Expand Down Expand Up @@ -1484,13 +1482,18 @@ private List<ResultsetRowData> fillDatatableResultSetDataRows(final String sql,
for (int i = 0; i < rsmd.getColumnCount(); i++) {
final String columnName = rsmd.getColumnName(i + 1);
final String colType = genericResultsetData.getColTypeOfColumnNamed(columnName);
if ("DECIMAL".equalsIgnoreCase(colType)) {
columnValues.add(rowSet.getBigDecimal(columnName));
} else if ("DATE".equalsIgnoreCase(colType) || ("TIMESTAMP WITHOUT TIME ZONE".equalsIgnoreCase(colType) // PostgreSQL
|| "DATETIME".equalsIgnoreCase(colType) || "TIMESTAMP".equalsIgnoreCase(colType))) {
columnValues.add(rowSet.getObject(columnName));

if ("DATE".equalsIgnoreCase(colType)) {
Date tmpDate = (Date) rowSet.getObject(columnName);
columnValues.add(tmpDate != null ? tmpDate.toLocalDate() : null);
} else if ("TIMESTAMP WITHOUT TIME ZONE".equalsIgnoreCase(colType) // PostgreSQL
|| "DATETIME".equalsIgnoreCase(colType) || "TIMESTAMP".equalsIgnoreCase(colType)) {
Object tmpDate = rowSet.getObject(columnName);

columnValues
.add(tmpDate != null ? tmpDate instanceof Timestamp ? ((Timestamp) tmpDate).toLocalDateTime() : tmpDate : null);
} else {
columnValues.add(rowSet.getString(columnName));
columnValues.add(rowSet.getObject(columnName));
}
}

Expand All @@ -1506,7 +1509,7 @@ private String queryForApplicationTableName(final String datatable) {

String applicationTableName = "";

final SqlRowSet rowSet = jdbcTemplate.queryForRowSet(sql, new Object[] { datatable }); // NOSONAR
final SqlRowSet rowSet = jdbcTemplate.queryForRowSet(sql, datatable); // NOSONAR
if (rowSet.next()) {
applicationTableName = rowSet.getString("application_table_name");
} else {
Expand All @@ -1523,7 +1526,7 @@ private String getFKField(final String applicationTableName) {
private String getAddSql(final List<ResultsetColumnHeaderData> columnHeaders, final String datatable, final String fkName,
final Long appTableId, final Map<String, String> queryParams) {

final Map<String, String> affectedColumns = getAffectedColumns(columnHeaders, queryParams, fkName);
final Map<String, Object> affectedColumns = getAffectedColumns(columnHeaders, queryParams, fkName);

String pValueWrite = "";
String addSql = "";
Expand All @@ -1536,7 +1539,7 @@ private String getAddSql(final List<ResultsetColumnHeaderData> columnHeaders, fi
for (final ResultsetColumnHeaderData pColumnHeader : columnHeaders) {
final String key = pColumnHeader.getColumnName();
if (affectedColumns.containsKey(key)) {
pValue = affectedColumns.get(key);
pValue = String.valueOf(affectedColumns.get(key));
if (StringUtils.isEmpty(pValue)) {
pValueWrite = "null";
} else {
Expand Down Expand Up @@ -1589,7 +1592,7 @@ private String getAddSql(final List<ResultsetColumnHeaderData> columnHeaders, fi
public String getAddSqlWithScore(final List<ResultsetColumnHeaderData> columnHeaders, final String datatable, final String fkName,
final Long appTableId, final Map<String, String> queryParams) {

final Map<String, String> affectedColumns = getAffectedColumns(columnHeaders, queryParams, fkName);
final Map<String, Object> affectedColumns = getAffectedColumns(columnHeaders, queryParams, fkName);

String pValueWrite = "";
String scoresId = " ";
Expand All @@ -1600,7 +1603,7 @@ public String getAddSqlWithScore(final List<ResultsetColumnHeaderData> columnHea
String columnName = "";
String pValue = null;
for (final String key : affectedColumns.keySet()) {
pValue = affectedColumns.get(key);
pValue = String.valueOf(affectedColumns.get(key));

if (StringUtils.isEmpty(pValue)) {
pValueWrite = "null";
Expand Down Expand Up @@ -1655,7 +1658,7 @@ private String getUpdateSql(List<ResultsetColumnHeaderData> columnHeaders, final
sql += ", ";
}

pValue = (String) changedColumns.get(key);
pValue = String.valueOf(changedColumns.get(key));
if (StringUtils.isEmpty(pValue)) {
pValueWrite = "null";
} else {
Expand Down Expand Up @@ -1688,41 +1691,37 @@ private String getUpdateSql(List<ResultsetColumnHeaderData> columnHeaders, final
private Map<String, Object> getAffectedAndChangedColumns(final GenericResultsetData grs, final Map<String, String> queryParams,
final String fkName) {

final Map<String, String> affectedColumns = getAffectedColumns(grs.getColumnHeaders(), queryParams, fkName);
final Map<String, Object> affectedColumns = getAffectedColumns(grs.getColumnHeaders(), queryParams, fkName);
final Map<String, Object> affectedAndChangedColumns = new HashMap<>();

for (final String key : affectedColumns.keySet()) {
final String columnValue = affectedColumns.get(key);
final String colType = grs.getColTypeOfColumnNamed(key);
if (columnChanged(key, columnValue, colType, grs)) {
final Object columnValue = affectedColumns.get(key);
if (columnChanged(key, columnValue, grs)) {
affectedAndChangedColumns.put(key, columnValue);
}
}

return affectedAndChangedColumns;
}

private boolean columnChanged(final String key, final String keyValue, final String colType, final GenericResultsetData grs) {
private boolean columnChanged(final String key, final Object value, final GenericResultsetData grs) {

final List<Object> columnValues = grs.getData().get(0).getRow();

String columnValue = null;
Object columnValue = null;
for (int i = 0; i < grs.getColumnHeaders().size(); i++) {

if (key.equals(grs.getColumnHeaders().get(i).getColumnName())) {
columnValue = (String) columnValues.get(i);
columnValue = columnValues.get(i);

if (notTheSame(columnValue, keyValue, colType)) {
return true;
}
return false;
return notTheSame(columnValue, value);
}
}

throw new PlatformDataIntegrityException("error.msg.invalid.columnName", "Parameter Column Name: " + key + " not found");
}

public Map<String, String> getAffectedColumns(final List<ResultsetColumnHeaderData> columnHeaders,
public Map<String, Object> getAffectedColumns(final List<ResultsetColumnHeaderData> columnHeaders,
final Map<String, String> queryParams, final String keyFieldName) {

final String dateFormat = queryParams.get("dateFormat");
Expand All @@ -1735,11 +1734,12 @@ public Map<String, String> getAffectedColumns(final List<ResultsetColumnHeaderDa
final String underscore = "_";
final String space = " ";
String pValue = null;
Object validatedValue = null;
String queryParamColumnUnderscored;
String columnHeaderUnderscored;
boolean notFound;

final Map<String, String> affectedColumns = new HashMap<>();
final Map<String, Object> affectedColumns = new HashMap<>();
final Set<String> keys = queryParams.keySet();
for (final String key : keys) {
// ignores id and foreign key fields
Expand All @@ -1756,8 +1756,8 @@ public Map<String, String> getAffectedColumns(final List<ResultsetColumnHeaderDa
columnHeaderUnderscored = this.genericDataService.replace(columnHeader.getColumnName(), space, underscore);
if (queryParamColumnUnderscored.equalsIgnoreCase(columnHeaderUnderscored)) {
pValue = queryParams.get(key);
pValue = validateColumn(columnHeader, pValue, dateFormat, clientApplicationLocale);
affectedColumns.put(columnHeader.getColumnName(), pValue);
validatedValue = validateColumn(columnHeader, pValue, dateFormat, clientApplicationLocale);
affectedColumns.put(columnHeader.getColumnName(), validatedValue);
notFound = false;
}
}
Expand All @@ -1771,7 +1771,7 @@ public Map<String, String> getAffectedColumns(final List<ResultsetColumnHeaderDa
return affectedColumns;
}

private String validateColumn(final ResultsetColumnHeaderData columnHeader, final String pValue, final String dateFormat,
private Object validateColumn(final ResultsetColumnHeaderData columnHeader, final String pValue, final String dateFormat,
final Locale clientApplicationLocale) {

String paramValue = pValue;
Expand All @@ -1793,7 +1793,6 @@ private String validateColumn(final ResultsetColumnHeaderData columnHeader, fina
}

if (StringUtils.isNotEmpty(paramValue)) {

if (columnHeader.hasColumnValues()) {
if (columnHeader.isCodeValueDisplayType()) {

Expand All @@ -1819,43 +1818,21 @@ private String validateColumn(final ResultsetColumnHeaderData columnHeader, fina
dataValidationErrors);
}

return paramValue;
return codeLookup;
} else {
throw new PlatformDataIntegrityException("error.msg.invalid.columnType.", "Code: " + columnHeader.getColumnName()
+ " - Invalid Type " + columnHeader.getColumnType() + " (neither varchar nor int)");
}
}

if (columnHeader.isDateDisplayType()) {
final LocalDate tmpDate = JsonParserHelper.convertFrom(paramValue, columnHeader.getColumnName(), dateFormat,
clientApplicationLocale);
if (tmpDate == null) {
paramValue = null;
} else {
paramValue = tmpDate.toString();
}
return JsonParserHelper.convertFrom(paramValue, columnHeader.getColumnName(), dateFormat, clientApplicationLocale);
} else if (columnHeader.isDateTimeDisplayType()) {
final LocalDateTime tmpDateTime = JsonParserHelper.convertDateTimeFrom(paramValue, columnHeader.getColumnName(), dateFormat,
clientApplicationLocale);
if (tmpDateTime == null) {
paramValue = null;
} else {
paramValue = tmpDateTime.toString();
}
return JsonParserHelper.convertDateTimeFrom(paramValue, columnHeader.getColumnName(), dateFormat, clientApplicationLocale);
} else if (columnHeader.isIntegerDisplayType()) {
final Integer tmpInt = this.helper.convertToInteger(paramValue, columnHeader.getColumnName(), clientApplicationLocale);
if (tmpInt == null) {
paramValue = null;
} else {
paramValue = tmpInt.toString();
}
return this.helper.convertToInteger(paramValue, columnHeader.getColumnName(), clientApplicationLocale);
} else if (columnHeader.isDecimalDisplayType()) {
final BigDecimal tmpDecimal = this.helper.convertFrom(paramValue, columnHeader.getColumnName(), clientApplicationLocale);
if (tmpDecimal == null) {
paramValue = null;
} else {
paramValue = tmpDecimal.toString();
}
return this.helper.convertFrom(paramValue, columnHeader.getColumnName(), clientApplicationLocale);
} else if (columnHeader.isBooleanDisplayType()) {

final Boolean tmpBoolean = BooleanUtils.toBooleanObject(paramValue);
Expand All @@ -1870,7 +1847,7 @@ private String validateColumn(final ResultsetColumnHeaderData columnHeader, fina
throw new PlatformApiDataValidationException("validation.msg.validation.errors.exist", "Validation errors exist.",
dataValidationErrors);
}
paramValue = tmpBoolean.toString();
return tmpBoolean;
} else if (columnHeader.isString()) {
if (columnHeader.getColumnLength() > 0 && paramValue.length() > columnHeader.getColumnLength()) {
final ApiParameterError error = ApiParameterError.parameterError(
Expand All @@ -1884,7 +1861,6 @@ private String validateColumn(final ResultsetColumnHeaderData columnHeader, fina
}
}
}

return paramValue;
}

Expand All @@ -1911,31 +1887,18 @@ private boolean isMultirowDatatable(final List<ResultsetColumnHeaderData> column
return multiRow;
}

private boolean notTheSame(final String currValue, final String pValue, final String colType) {
if (StringUtils.isEmpty(currValue) && StringUtils.isEmpty(pValue)) {
private boolean notTheSame(final Object currValue, final Object pValue) {
if (currValue == null && pValue == null) {
return false;
}

if (StringUtils.isEmpty(currValue)) {
return true;
}

if (StringUtils.isEmpty(pValue)) {
} else if (currValue == null || pValue == null) {
return true;
}

if ("DECIMAL".equalsIgnoreCase(colType)) {
final BigDecimal currentDecimal = BigDecimal.valueOf(Double.parseDouble(currValue));
final BigDecimal newDecimal = BigDecimal.valueOf(Double.parseDouble(pValue));

return currentDecimal.compareTo(newDecimal) != 0;
}

if (currValue.equals(pValue)) {
return false;
// Equals would fail if the scale is not the same
if (currValue instanceof BigDecimal && pValue instanceof BigDecimal) {
return !(((BigDecimal) currValue).compareTo((BigDecimal) pValue) == 0);
} else {
return !currValue.equals(pValue);
}

return true;
}

@Override
Expand Down
Loading