Skip to content

FINERACT-1910: Added pagination & sorting support in datatables/query #3076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shyam1s15 wants to merge 1 commit into from
Closed

FINERACT-1910: Added pagination & sorting support in datatables/query #3076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion ...a/org/apache/fineract/batch/command/internal/GetDatatableEntryByQueryCommandStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,14 @@ public BatchResponse execute(final BatchRequest request, @SuppressWarnings("unus
// - Call datatablesApiResource.getDatatable(dataTable, appTableId, null, uriInfo)
String columnFilter = null;
String valueFilter = null;
String dateFilter = null;
String likeFilter = null;
String resultColumns = null;
Integer limit = null;
Integer offset = null;
String orderBy = null;
String sortOrder = null;

if (relativeUrl.indexOf('?') > 0) {
Map<String, String> queryParameters = CommandStrategyUtils.getQueryParameters(relativeUrl);
// Add the query parameters sent in the relative URL to MutableUriInfo
Expand All @@ -78,7 +85,13 @@ public BatchResponse execute(final BatchRequest request, @SuppressWarnings("unus
switch (entry.getKey()) {
case "columnFilter" -> columnFilter = entry.getValue();
case "valueFilter" -> valueFilter = entry.getValue();
case "dateFilter" -> dateFilter = entry.getValue();
case "likeFilter" -> likeFilter = entry.getValue();
case "resultColumns" -> resultColumns = entry.getValue();
case "limit" -> limit = Integer.valueOf(entry.getValue());
case "offset" -> offset = Integer.valueOf(entry.getValue());
case "orderBy" -> orderBy = entry.getValue();
case "sortOrder" -> sortOrder = entry.getValue();
}
}
}
Expand All @@ -87,7 +100,8 @@ public BatchResponse execute(final BatchRequest request, @SuppressWarnings("unus

// Calls 'queryValues' function from 'DatatablesApiResource' to
// get the datatable details based on the filters
responseBody = dataTablesApiResource.queryValues(dataTableName, columnFilter, valueFilter, resultColumns, parameterizedUriInfo);
responseBody = dataTablesApiResource.queryValues(dataTableName, columnFilter, valueFilter, dateFilter, likeFilter, resultColumns,
offset, limit, orderBy, sortOrder, parameterizedUriInfo);

response.setStatusCode(HttpStatus.SC_OK);

Expand Down
11 changes: 8 additions & 3 deletions ...c/main/java/org/apache/fineract/infrastructure/dataqueries/api/DatatablesApiResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,12 +228,17 @@ public String getDatatable(@PathParam("datatable") @Parameter(description = "dat
public String queryValues(@PathParam("datatable") @Parameter(description = "datatable") final String datatable,
@QueryParam("columnFilter") @Parameter(description = "columnFilter") final String columnFilter,
@QueryParam("valueFilter") @Parameter(description = "valueFilter") final String valueFilter,
@QueryParam("dateFilter") @Parameter(description = "dateFilter") final String dateFilter,
@QueryParam("likeFilter") @Parameter(description = "likeFilter") final String likeFilter,
@QueryParam("resultColumns") @Parameter(description = "resultColumns") final String resultColumns,
@Context final UriInfo uriInfo) {
@QueryParam("offset") @Parameter(description = "offset") final Integer offset,
@QueryParam("limit") @Parameter(description = "limit") final Integer limit,
@QueryParam("orderBy") @Parameter(description = "orderBy") final String orderBy,
@QueryParam("sortOrder") @Parameter(description = "sortOrder") final String sortOrder, @Context final UriInfo uriInfo) {
this.context.authenticatedUser().validateHasDatatableReadPermission(datatable);

final List<JsonObject> result = this.readWriteNonCoreDataService.queryDataTable(datatable, columnFilter, valueFilter,
resultColumns);
final List<JsonObject> result = this.readWriteNonCoreDataService.queryDataTable(datatable, columnFilter, valueFilter, dateFilter,
likeFilter, resultColumns, offset, limit, orderBy, sortOrder);

return this.toApiJsonSerializer.serializePretty(ApiParameterHelper.prettyPrint(uriInfo.getQueryParameters()), result);
}
Expand Down
8 changes: 7 additions & 1 deletion ...a/org/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,13 @@ public interface ReadWriteNonCoreDataService {

DatatableData retrieveDatatable(String datatable);

List<JsonObject> queryDataTable(String datatable, String columnFilter, String valueFilter, String resultColumns);
/**
* @Description This method will find data from specified table, and return specified column using parameter
* resultColumns There are many filters used in this query which can help to generate sql query from
* api parameters
*/
List<JsonObject> queryDataTable(String datatable, String columnFilter, String valueFilter, String dateFilter, String likeFilter,
String resultColumns, Integer offset, Integer limit, String orderBy, String sortOrder);

@PreAuthorize(value = "hasAnyAuthority('ALL_FUNCTIONS', 'REGISTER_DATATABLE')")
void registerDatatable(JsonCommand command);
Expand Down
101 changes: 82 additions & 19 deletions ...g/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
import javax.persistence.PersistenceException;
Expand Down Expand Up @@ -204,24 +203,76 @@ public DatatableData retrieveDatatable(final String datatable) {
}

@Override
public List<JsonObject> queryDataTable(String datatable, String columnFilter, String valueFilter, String resultColumns) {
Arrays.asList(datatable, columnFilter, valueFilter, resultColumns).forEach(SQLInjectionValidator::validateDynamicQuery);
public List<JsonObject> queryDataTable(String datatable, String columnFilter, String valueFilter, String dateFilter, String likeFilter,
String resultColumns, Integer offset, Integer limit, String orderBy, String sortOrder) {

Arrays.asList(datatable, columnFilter, valueFilter, resultColumns, orderBy, sortOrder)
.forEach(SQLInjectionValidator::validateDynamicQuery);

List<ResultsetColumnHeaderData> resultsetColumnHeaderData = genericDataService.fillResultsetColumnHeaders(datatable);
validateRequestParams(columnFilter, valueFilter, resultColumns, resultsetColumnHeaderData);

String sql = "select " + resultColumns + " from " + datatable + " where " + columnFilter + " = ?";
SqlRowSet rowSet = null;
String filterColumnType = resultsetColumnHeaderData.stream().filter(column -> Objects.equals(columnFilter, column.getColumnName()))
.findFirst().map(ResultsetColumnHeaderData::getColumnType).orElse(columnFilter + " does not exist in datatable");
if (databaseTypeResolver.isPostgreSQL()) {
rowSet = callFilteredPgSql(sql, valueFilter, filterColumnType);
} else if (databaseTypeResolver.isMySQL()) {
rowSet = callFilteredMysql(sql, valueFilter, filterColumnType);
} else {
throw new IllegalStateException("Database type is not supported");
validateRequestParams(columnFilter, valueFilter, likeFilter, resultColumns, resultsetColumnHeaderData, orderBy, sortOrder);

List<Object> paramList = new ArrayList<>();
StringBuilder sql = new StringBuilder("select " + resultColumns + " from " + datatable + " where ");

// value search must not be null or like filter must not be null
if (StringUtils.isNotBlank(columnFilter) && (StringUtils.isNotBlank(valueFilter) || StringUtils.isNotBlank(likeFilter))) {
// priority given to like search
if (StringUtils.isNotBlank(likeFilter)) {
sql.append(columnFilter).append(" LIKE '%").append(likeFilter).append("%' ");
paramList.add(likeFilter);
} else {
sql.append(columnFilter).append(" = ?");
paramList.add(valueFilter);
}
}

if (StringUtils.isNotBlank(dateFilter)) {
LocalDate from = null;
LocalDate to = null;

String[] dates = dateFilter.split("-");
String fromDate = dates.length > 0 ? dates[0].split("_")[1] : null;
String toDate = dates.length > 1 ? dates[1].split("_")[1] : null;

DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
if (fromDate != null) {
from = LocalDate.parse(fromDate, formatter);
}
if (toDate != null) {
to = LocalDate.parse(toDate, formatter);
}

if (from != null) {
if (to != null) {
sql.append(" and ").append(columnFilter).append("between ? and ? ");
paramList.add(java.sql.Date.valueOf(to));
paramList.add(java.sql.Date.valueOf(from));
} else {
sql.append(" and ").append(columnFilter).append(" > ?");
paramList.add(java.sql.Date.valueOf(from));
}
}
}

if (StringUtils.isNotBlank(orderBy)) {
sql.append(" order by ? ");
paramList.add(orderBy);
}
if (StringUtils.isNotBlank(sortOrder)) {
sql.append(" ").append(sortOrder).append(" ");
paramList.add(sortOrder);
}
if (limit != null) {
if (offset != null) {
sql.append(sqlGenerator.limit(limit, offset));
} else {
sql.append(sqlGenerator.limit(limit));
}
}

final SqlRowSet rowSet = jdbcTemplate.queryForRowSet(sql.toString(), paramList); // NOSONAR

String[] resultColumnNames = resultColumns.split(",");
List<JsonObject> results = new ArrayList<>();
while (rowSet.next()) {
Expand Down Expand Up @@ -332,8 +383,8 @@ private SqlRowSet callFilteredPgSql(String sql, String valueFilter, String filte
return jdbcTemplate.queryForRowSet(sql, new Object[] { finalValueFilter }, argType);
}

private static void validateRequestParams(String columnFilter, String valueFilter, String resultColumns,
List<ResultsetColumnHeaderData> resultsetColumnHeaderData) {
private static void validateRequestParams(String columnFilter, String valueFilter, String likeFilter, String resultColumns,
List<ResultsetColumnHeaderData> resultsetColumnHeaderData, String orderBy, String sortOrder) {
List<ApiParameterError> paramErrors = new ArrayList<>();
List<String> dataTableColumnNames = resultsetColumnHeaderData.stream().map(ResultsetColumnHeaderData::getColumnName).toList();
if (columnFilter == null || columnFilter.isEmpty()) {
Expand All @@ -344,8 +395,9 @@ private static void validateRequestParams(String columnFilter, String valueFilte
}
}

if (valueFilter == null || valueFilter.isEmpty()) {
paramErrors.add(parameterErrorWithValue("400", "Value filter is empty!", "valueFilter", valueFilter));
if ((valueFilter == null || valueFilter.isEmpty() && (likeFilter == null || likeFilter.isEmpty()))) {
paramErrors.add(parameterErrorWithValue("400", "Value filter & likw filter is empty!", "valueFilter & likeFilter",
valueFilter + " " + likeFilter));
}

if (resultColumns == null || resultColumns.isEmpty()) {
Expand All @@ -358,6 +410,17 @@ private static void validateRequestParams(String columnFilter, String valueFilte
});
}

if (StringUtils.isNotBlank(orderBy)) {
if (!dataTableColumnNames.contains(orderBy)) {
paramErrors.add(parameterErrorWithValue("400", "Column orderBy not exist in datatable!", "orderBy", orderBy));
}
}
if (StringUtils.isNotBlank(sortOrder)) {
if (!dataTableColumnNames.contains(sortOrder)) {
paramErrors.add(parameterErrorWithValue("400", "Column sortOrder not exist in datatable!", "sortOrder", sortOrder));
}
}

if (!paramErrors.isEmpty()) {
throw new PlatformApiDataValidationException(paramErrors);
}
Expand Down
11 changes: 3 additions & 8 deletions ...rovider/src/main/java/org/apache/fineract/portfolio/loanaccount/api/LoansApiResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -844,7 +844,9 @@ private String retrieveLoan(final Long loanId, final String loanExternalIdStr, b
Collection<LoanCollateralResponseData> loanCollateralManagements;
Collection<LoanCollateralManagementData> loanCollateralManagementData = new ArrayList<>();
CollectionData collectionData = CollectionData.template();

if (loanBasicDetails.isActive()) {
collectionData = this.delinquencyReadPlatformService.calculateLoanCollectionData(resolvedLoanId);
}
final Set<String> mandatoryResponseParameters = new HashSet<>();
final Set<String> associationParameters = ApiParameterHelper.extractAssociationsForResponseIfProvided(uriInfo.getQueryParameters());
final Collection<LoanTransactionData> currentLoanRepayments = this.loanReadPlatformService.retrieveLoanTransactions(resolvedLoanId);
Expand Down Expand Up @@ -948,13 +950,6 @@ private String retrieveLoan(final Long loanId, final String loanExternalIdStr, b
mandatoryResponseParameters.add(DataTableApiConstant.linkedAccountAssociateParamName);
linkedAccount = this.accountAssociationsReadPlatformService.retriveLoanLinkedAssociation(resolvedLoanId);
}

if (associationParameters.contains(DataTableApiConstant.collectionAssociateParamName)) {
mandatoryResponseParameters.add(DataTableApiConstant.collectionAssociateParamName);
if (loanBasicDetails.isActive()) {
collectionData = this.delinquencyReadPlatformService.calculateLoanCollectionData(resolvedLoanId);
}
}
}

Collection<LoanProductData> productOptions = null;
Expand Down
14 changes: 9 additions & 5 deletions ...ache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,15 @@ public void setup() {
@Test
public void testSqlInjectionCaughtQueryDataTable() {
assertThrows(SQLInjectionException.class, () -> {
underTest.queryDataTable("table", "cf1", "vf1", "' or 1=1");
underTest.queryDataTable("table", "cf1", "vf1", "from_20210201-to_20230228", null, "' or 1=1", 0, 10, "", "");
});
}

@Test
public void testSqlInjectionCaughtQueryDataTable2() {
assertThrows(SQLInjectionException.class, () -> {
underTest.queryDataTable("table", "cf1", "vf1", "1; DROP TABLE m_loan; SELECT");
underTest.queryDataTable("table", "cf1", "vf1", "from_20210201-to_20230228", null, "1; DROP TABLE m_loan; SELECT", 0, 10, "",
"");
});
}

Expand All @@ -88,7 +89,8 @@ public void testQueryDataTableSuccess() {
ResultsetColumnHeaderData rc2 = ResultsetColumnHeaderData.detailed("rc2", "text", 10L, false, false, null, null, false, false);
when(genericDataService.fillResultsetColumnHeaders("table")).thenReturn(List.of(cf1, rc1, rc2));

List<JsonObject> results = underTest.queryDataTable("table", "cf1", "vf1", "rc1,rc2");
List<JsonObject> results = underTest.queryDataTable("table", "cf1", "vf1", "from_20210201-to_20230228", null, "rc1,rc2", 0, 10, "",
"");

Assertions.assertEquals("value1", results.get(0).get("rc1").getAsString());
Assertions.assertEquals("value2", results.get(0).get("rc2").getAsString());
Expand All @@ -97,7 +99,8 @@ public void testQueryDataTableSuccess() {
@Test
public void testQueryDataTableValidationError() {
when(genericDataService.fillResultsetColumnHeaders("table")).thenReturn(Collections.emptyList());
assertThrows(PlatformApiDataValidationException.class, () -> underTest.queryDataTable("table", "cf1", "vf1", "rc1,rc2"));
assertThrows(PlatformApiDataValidationException.class,
() -> underTest.queryDataTable("table", "cf1", "vf1", "from_20210201-to_20230228", null, "rc1,rc2", 0, 10, "", ""));
}

@Test
Expand All @@ -114,6 +117,7 @@ public void testInvalidDatabase() {
ResultsetColumnHeaderData rc2 = ResultsetColumnHeaderData.detailed("rc2", "text", 10L, false, false, null, null, false, false);
when(genericDataService.fillResultsetColumnHeaders("table")).thenReturn(List.of(cf1, rc1, rc2));

assertThrows(IllegalStateException.class, () -> underTest.queryDataTable("table", "cf1", "vf1", "rc1,rc2"));
assertThrows(IllegalStateException.class,
() -> underTest.queryDataTable("table", "cf1", "vf1", "from_20210201-to_20230228", null, "rc1,rc2", 0, 10, "", ""));
}
}