Skip to content

Fix the datasync upload issue #107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 8, 2025
Merged

Fix the datasync upload issue #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ac60345
fix: add the schemas
vanitha1822 Aug 6, 2025
767bf17
fix: remove logger
vanitha1822 Aug 6, 2025
6479542
fix: revert the old code for repository
vanitha1822 Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public List<Map<String, Object>> getDataForGivenSchemaAndTable(String schema, St

if (table != null && table.equalsIgnoreCase("m_beneficiaryregidmapping")) {
baseQuery = " SELECT " + columnNames + " FROM " + schema + "." + table
+ " WHERE provisioned is true AND processed != 'P' AND vanID is not null ";
+ " WHERE provisioned is true AND processed <> 'P' AND vanID is not null ";
} else {
if (table != null && (table.equalsIgnoreCase("t_patientissue")
|| table.equalsIgnoreCase("t_physicalstockentry") || table.equalsIgnoreCase("t_stockadjustment")
Expand All @@ -75,15 +75,14 @@ public List<Map<String, Object>> getDataForGivenSchemaAndTable(String schema, St
|| table.equalsIgnoreCase("t_itemstockexit"))) {

baseQuery = " SELECT " + columnNames + " FROM " + schema + "." + table
+ " WHERE processed != 'P' AND SyncFacilityID is not null ";
+ " WHERE processed <> 'P' AND SyncFacilityID is not null ";

} else {
baseQuery = " SELECT " + columnNames + " FROM " + schema + "." + table
+ " WHERE processed != 'P' AND vanID is not null ";
+ " WHERE processed <> 'P' AND vanID is not null ";
}

}

resultSetList = jdbcTemplate.queryForList(baseQuery);
return resultSetList;
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ public List<SyncUtilityClass> getVanAndServerColumnList(Integer groupID) throws

private List<Map<String, Object>> getDataToSync(String schemaName, String tableName, String columnNames)
throws Exception {
logger.info("Fetching data to sync for schema: {}, table: {}, columns: {}", schemaName, tableName, columnNames);
List<Map<String, Object>> resultSetList = dataSyncRepository.getDataForGivenSchemaAndTable(schemaName,
tableName, columnNames);
if (resultSetList != null) {
Expand Down
234 changes: 112 additions & 122 deletions src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,19 @@
*/
package com.iemr.mmu.service.dataSyncLayerCentral;

import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.sql.DataSource;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;

import com.iemr.mmu.data.syncActivity_syncLayer.SyncUploadDataDigester;
import javax.sql.DataSource;
import java.sql.Timestamp;
import java.sql.Statement; // Import Statement for batchUpdate result interpretation
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

@Service
public class DataSyncRepositoryCentral {
Expand All @@ -44,177 +42,169 @@ public class DataSyncRepositoryCentral {

private JdbcTemplate jdbcTemplate;

// Lazily initialize jdbcTemplate to ensure DataSource is available
private JdbcTemplate getJdbcTemplate() {
if (this.jdbcTemplate == null) {
this.jdbcTemplate = new JdbcTemplate(dataSource);
}
return this.jdbcTemplate;
}

private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName());

private static final Set<String> VALID_SCHEMAS = Set.of("public", "db_iemr");

private static final Set<String> VALID_TABLES = Set.of(
"m_beneficiaryregidmapping", "i_beneficiaryaccount", "i_beneficiaryaddress", "i_beneficiarycontacts",
"i_beneficiarydetails", "i_beneficiaryfamilymapping", "i_beneficiaryidentity", "i_beneficiarymapping",
"t_benvisitdetail", "t_phy_anthropometry", "t_phy_vitals", "t_benadherence", "t_anccare", "t_pnccare",
"t_ncdscreening", "t_ncdcare", "i_ben_flow_outreach", "t_covid19", "t_idrsdetails", "t_physicalactivity",
"t_phy_generalexam", "t_phy_headtotoe", "t_sys_obstetric", "t_sys_gastrointestinal", "t_sys_cardiovascular",
"t_sys_respiratory", "t_sys_centralnervous", "t_sys_musculoskeletalsystem", "t_sys_genitourinarysystem",
"t_ancdiagnosis", "t_ncddiagnosis", "t_pncdiagnosis", "t_benchefcomplaint", "t_benclinicalobservation",
"t_prescription", "t_prescribeddrug", "t_lab_testorder", "t_benreferdetails",
"t_lab_testresult", "t_physicalstockentry", "t_patientissue", "t_facilityconsumption", "t_itemstockentry",
"t_itemstockexit", "t_benmedhistory", "t_femaleobstetrichistory", "t_benmenstrualdetails",
"t_benpersonalhabit", "t_childvaccinedetail1", "t_childvaccinedetail2", "t_childoptionalvaccinedetail",
"t_ancwomenvaccinedetail", "t_childfeedinghistory", "t_benallergyhistory", "t_bencomorbiditycondition",
"t_benmedicationhistory", "t_benfamilyhistory", "t_perinatalhistory", "t_developmenthistory",
"t_cancerfamilyhistory", "t_cancerpersonalhistory", "t_cancerdiethistory", "t_cancerobstetrichistory",
"t_cancervitals", "t_cancersignandsymptoms", "t_cancerlymphnode", "t_canceroralexamination",
"t_cancerbreastexamination", "t_cancerabdominalexamination", "t_cancergynecologicalexamination",
"t_cancerdiagnosis", "t_cancerimageannotation", "i_beneficiaryimage", "t_stockadjustment",
"t_stocktransfer", "t_patientreturn", "t_indent", "t_indentissue", "t_indentorder", "t_saitemmapping"
);

private boolean isValidDatabaseIdentifierCharacter(String identifier) {
return identifier != null && identifier.matches("^[a-zA-Z_][a-zA-Z0-9_]*$");
}

private boolean isValidSchemaName(String schemaName) {
return VALID_SCHEMAS.contains(schemaName.toLowerCase());
}

private boolean isValidTableName(String tableName) {
return VALID_TABLES.contains(tableName.toLowerCase());
}

private boolean isValidColumnNamesList(String columnNames) {
if (columnNames == null || columnNames.trim().isEmpty()) {
return false;
}
for (String col : columnNames.split(",")) {
if (!isValidDatabaseIdentifierCharacter(col.trim())) {
return false;
}
}
return true;
}
private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName());

// Data Upload Repository
public int checkRecordIsAlreadyPresentOrNot(String schemaName, String tableName, String vanSerialNo, String vanID,
String vanAutoIncColumnName, int syncFacilityID) {
jdbcTemplate = getJdbcTemplate();
List<Object> params = new ArrayList<>();

if (!isValidSchemaName(schemaName) || !isValidTableName(tableName) ||
!isValidDatabaseIdentifierCharacter(vanAutoIncColumnName)) {
logger.error("Invalid identifiers: schema={}, table={}, column={}", schemaName, tableName, vanAutoIncColumnName);
throw new IllegalArgumentException("Invalid identifiers provided.");
}
List<Object> params = new ArrayList<>();

StringBuilder queryBuilder = new StringBuilder("SELECT ")
.append(vanAutoIncColumnName).append(" FROM ")
.append(schemaName).append(".").append(tableName).append(" WHERE VanSerialNo = ?");
StringBuilder queryBuilder = new StringBuilder("SELECT ");
queryBuilder.append(vanAutoIncColumnName);
queryBuilder.append(" FROM ");
queryBuilder.append(schemaName).append(".").append(tableName);

StringBuilder whereClause = new StringBuilder();
whereClause.append(" WHERE ");
whereClause.append("VanSerialNo = ?");
params.add(vanSerialNo);

if (List.of("t_patientissue", "t_physicalstockentry", "t_stockadjustment", "t_saitemmapping",
if (Arrays.asList("t_patientissue", "t_physicalstockentry", "t_stockadjustment", "t_saitemmapping",
"t_stocktransfer", "t_patientreturn", "t_facilityconsumption", "t_indent",
"t_indentorder", "t_indentissue", "t_itemstockentry", "t_itemstockexit").contains(tableName.toLowerCase()) && syncFacilityID > 0) {
queryBuilder.append(" AND SyncFacilityID = ?");
"t_indentorder", "t_indentissue", "t_itemstockentry", "t_itemstockexit")
.contains(tableName.toLowerCase()) && syncFacilityID > 0) {

whereClause.append(" AND ");
whereClause.append("SyncFacilityID = ?");
params.add(syncFacilityID);

} else {
queryBuilder.append(" AND VanID = ?");
whereClause.append(" AND ");
whereClause.append("VanID = ?");
params.add(vanID);
}

queryBuilder.append(whereClause);
String query = queryBuilder.toString();
Object[] queryParams = params.toArray();

logger.debug("Checking record existence query: {} with params: {}", query, Arrays.toString(queryParams));
System.out.println("Checking record existence query: " + query + " with params: " + Arrays.toString(queryParams));

try {
List<Map<String, Object>> resultSet = jdbcTemplate.queryForList(queryBuilder.toString(), params.toArray());
return (resultSet != null && !resultSet.isEmpty()) ? 1 : 0;
List<Map<String, Object>> resultSet = jdbcTemplate.queryForList(query, queryParams);
if (resultSet != null && !resultSet.isEmpty()) {
System.out.println("Record found for table " + tableName + ": VanSerialNo=" + vanSerialNo + ", VanID=" + vanID);
logger.debug("Record found for table {}: VanSerialNo={}, VanID={}", tableName, vanSerialNo, vanID);
return 1;
} else {
System.out.println("No record found for table " + tableName + ": VanSerialNo=" + vanSerialNo + ", VanID=" + vanID);
logger.debug("No record found for table {}: VanSerialNo={}, VanID={}", tableName, vanSerialNo, vanID);
return 0;
}
} catch (org.springframework.dao.EmptyResultDataAccessException e) {
System.out.println("No record found (EmptyResultDataAccessException) for table " + tableName + ": VanSerialNo=" + vanSerialNo + ", VanID=" + vanID);
logger.debug("No record found (EmptyResultDataAccessException) for table {}: VanSerialNo={}, VanID={}", tableName, vanSerialNo, vanID);
return 0;
} catch (Exception e) {
logger.error("Error checking record presence: {}", e.getMessage(), e);
throw new RuntimeException("Failed to check record existence: " + e.getMessage(), e);
System.out.println("Database error during checkRecordIsAlreadyPresentOrNot for table " + tableName + ": VanSerialNo=" + vanSerialNo + ", VanID=" + vanID);
logger.error("Database error during checkRecordIsAlreadyPresentOrNot for table {}: VanSerialNo={}, VanID={}. Error: {}", tableName, vanSerialNo, vanID, e.getMessage(), e);
throw new RuntimeException("Failed to check record existence: " + e.getMessage(), e); // Re-throw or handle as appropriate
}
}

// Method for synchronization of data to central DB
public int[] syncDataToCentralDB(String schema, String tableName, String serverColumns, String query,
List<Object[]> syncDataList) {
jdbcTemplate = getJdbcTemplate();
logger.info("Executing batch operation for table: {}. Query type: {}. Number of records: {}", tableName, query.startsWith("INSERT") ? "INSERT" : "UPDATE", syncDataList.size());
logger.debug("Query: {}", query);
System.out.println("Executing batch operation for table: " + tableName + ". Query type: " + (query.startsWith("INSERT") ? "INSERT" : "UPDATE") + ". Number of records: " + syncDataList.size());
try {
return jdbcTemplate.batchUpdate(query, syncDataList);
// Start batch insert/update
int[] i = jdbcTemplate.batchUpdate(query, syncDataList);
System.out.println("Batch operation completed for table " + tableName + ". Results: " + Arrays.toString(i));
logger.info("Batch operation completed for table {}. Results: {}", tableName, Arrays.toString(i));
return i;
} catch (Exception e) {
logger.error("Batch sync failed for table {}: {}", tableName, e.getMessage(), e);
throw new RuntimeException("Batch sync failed: " + e.getMessage(), e);
logger.error("Exception during batch update for table {}: {}", tableName, e.getMessage(), e);
System.out.println("Exception during batch update for table " + tableName + ": " + e.getMessage());
// Log the error with detailed information
// Re-throw the exception to be handled by the service layer, so specific errors can be captured.
throw new RuntimeException("Batch sync failed for table " + tableName + ": " + e.getMessage(), e);
}
}

// End of Data Upload Repository

public List<Map<String, Object>> getMasterDataFromTable(String schema, String table, String columnNames,
String masterType, Timestamp lastDownloadDate, Integer vanID, Integer psmID) {
String masterType, Timestamp lastDownloadDate, Integer vanID, Integer psmID) throws Exception {
jdbcTemplate = getJdbcTemplate();
List<Map<String, Object>> resultSetList = new ArrayList<>();
StringBuilder baseQueryBuilder = new StringBuilder(" SELECT ").append(columnNames).append(" FROM ").append(schema).append(".").append(table);
List<Object> params = new ArrayList<>();

if (!isValidSchemaName(schema) || !isValidTableName(table) || !isValidColumnNamesList(columnNames)) {
throw new IllegalArgumentException("Invalid schema, table, or column names.");
}

StringBuilder queryBuilder = new StringBuilder("SELECT ").append(columnNames)
.append(" FROM ").append(schema).append(".").append(table);

if (masterType != null) {
if (lastDownloadDate != null) {
queryBuilder.append(" WHERE LastModDate >= ?");
baseQueryBuilder.append(" WHERE LastModDate >= ? ");
params.add(lastDownloadDate);

if ("V".equalsIgnoreCase(masterType)) {
queryBuilder.append(" AND VanID = ?");
if (masterType.equalsIgnoreCase("V")) {
baseQueryBuilder.append(" AND VanID = ? ");
params.add(vanID);
} else if ("P".equalsIgnoreCase(masterType)) {
queryBuilder.append(" AND ProviderServiceMapID = ?");
} else if (masterType.equalsIgnoreCase("P")) {
baseQueryBuilder.append(" AND ProviderServiceMapID = ? ");
params.add(psmID);
}
} else {
queryBuilder.append(" WHERE ");
if ("V".equalsIgnoreCase(masterType)) {
queryBuilder.append("VanID = ?");
if (masterType.equalsIgnoreCase("V")) {
baseQueryBuilder.append(" WHERE VanID = ? ");
params.add(vanID);
} else if ("P".equalsIgnoreCase(masterType)) {
queryBuilder.append("ProviderServiceMapID = ?");
} else if (masterType.equalsIgnoreCase("P")) {
baseQueryBuilder.append(" WHERE ProviderServiceMapID = ? ");
params.add(psmID);
}
}
}

String finalQuery = baseQueryBuilder.toString();
logger.info("Select query central: {}", finalQuery);
logger.info("Last Downloaded Date: {}", lastDownloadDate);
logger.info("Query Params: {}", params);
System.out.println("Select query central: " + finalQuery);
System.out.println("Last Downloaded Date: " + lastDownloadDate);
System.out.println("Query Params: " + params);

try {
// Safe dynamic SQL: All dynamic parts (table names, columns, etc.) are validated or hardcoded.
// Parameter values are bound safely using prepared statement placeholders (?).
return jdbcTemplate.queryForList(queryBuilder.toString(), params.toArray());
if (params.isEmpty()) {
resultSetList = jdbcTemplate.queryForList(finalQuery);
} else {
resultSetList = jdbcTemplate.queryForList(finalQuery, params.toArray());
}
} catch (Exception e) {
logger.error("Error fetching master data: {}", e.getMessage(), e);
System.out.println("Error fetching master data from table " + table + ": " + e.getMessage());
logger.error("Error fetching master data from table {}: {}", table, e.getMessage(), e);
throw new RuntimeException("Failed to fetch master data: " + e.getMessage(), e);
}
System.out.println("Result set Details size: " + resultSetList.size());
logger.info("Result set Details size: {}", resultSetList.size());
return resultSetList;
}

public List<Map<String, Object>> getBatchForBenDetails(SyncUploadDataDigester digester,
public List<Map<String, Object>> getBatchForBenDetails(String schema, String table, String columnNames,
String whereClause, int limit, int offset) {
jdbcTemplate = getJdbcTemplate();

String schema = digester.getSchemaName();
String table = digester.getTableName();
String columnNames = digester.getServerColumns();


if (!isValidSchemaName(schema) || !isValidTableName(table) || !isValidColumnNamesList(columnNames)) {
throw new IllegalArgumentException("Invalid schema, table, or column names.");
}
// Safe dynamic SQL: Schema, table, and column names are validated against predefined whitelists.
// Only trusted values are used in the query string.
// limit and offset are passed as parameters to prevent SQL injection.
String query = String.format("SELECT %s FROM %s.%s %s LIMIT ? OFFSET ?", columnNames, schema, table, whereClause); //NOSONAR

try {
return jdbcTemplate.queryForList(query, limit, offset);
} catch (Exception e) {
logger.error("Error fetching batch details: {}", e.getMessage(), e);
throw new RuntimeException("Failed to fetch batch data: " + e.getMessage(), e);
}
jdbcTemplate = getJdbcTemplate();
String query = "SELECT " + columnNames + " FROM " + schema + "." + table + whereClause + " LIMIT ? OFFSET ?";
System.out.println("Fetching batch for beneficiary details. Query: " + query + ", Limit: " + limit + ", Offset: " + offset);
logger.debug("Fetching batch for beneficiary details. Query: {}, Limit: {}, Offset: {}", query, limit, offset);
try {
return jdbcTemplate.queryForList(query, limit, offset);
} catch (Exception e) {
logger.error("Error fetching batch for beneficiary details from table {}: {}", table, e.getMessage(), e);
System.out.println("Error fetching batch for beneficiary details from table " + table + ": " + e.getMessage());
throw new RuntimeException("Failed to fetch batch data: " + e.getMessage(), e);
}
}
}

// End of Data Download Repository
}
Loading