Skip to content

Commit

Permalink
fix : quicken up loading 31stJan2018 NAV
Browse files Browse the repository at this point in the history
  • Loading branch information
rajadilipkolli committed Apr 29, 2024
1 parent faf9b27 commit ac2c5da
Show file tree
Hide file tree
Showing 8 changed files with 6,073 additions and 6,007 deletions.
16 changes: 10 additions & 6 deletions src/main/java/com/learning/mfscreener/config/Initializer.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import com.learning.mfscreener.entities.MFSchemeEntity;
import com.learning.mfscreener.mapper.MfSchemeDtoToEntityMapper;
import com.learning.mfscreener.models.MFSchemeDTO;
import com.learning.mfscreener.service.HistoricalNavService;
import com.learning.mfscreener.service.MFSchemeNavService;
import com.learning.mfscreener.service.SchemeService;
import com.learning.mfscreener.utils.AppConstants;
import java.io.BufferedReader;
Expand Down Expand Up @@ -31,17 +31,17 @@ public class Initializer implements CommandLineRunner {
private final SchemeService schemeService;
private final MfSchemeDtoToEntityMapper mfSchemeDtoToEntityMapper;
private final RestTemplate restTemplate;
private final HistoricalNavService historicalNavService;
private final MFSchemeNavService mfSchemeNavService;

public Initializer(
SchemeService schemeService,
MfSchemeDtoToEntityMapper mfSchemeDtoToEntityMapper,
RestTemplate restTemplate,
HistoricalNavService historicalNavService) {
MFSchemeNavService mfSchemeNavService) {
this.schemeService = schemeService;
this.mfSchemeDtoToEntityMapper = mfSchemeDtoToEntityMapper;
this.restTemplate = restTemplate;
this.historicalNavService = historicalNavService;
this.mfSchemeNavService = mfSchemeNavService;
}

@Override
Expand Down Expand Up @@ -120,8 +120,12 @@ public void run(String... args) throws IOException {
LOGGER.error("Unable to load data from :{}", AppConstants.AMFI_WEBSITE_LINK, httpClientErrorException);
}

if (!schemeService.navLoadedFor31Jan2018()) {
historicalNavService.getHistoricalNavOn31Jan2018();
if (!mfSchemeNavService.navLoadedFor31Jan2018ForExistingSchemes()) {
mfSchemeNavService.loadHistoricalNavOn31Jan2018ForExistingSchemes();
}

if (!mfSchemeNavService.navLoadedForClosedOrMergedSchemes()) {
schemeService.loadHistoricalDataForClosedOrMergedSchemes();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,13 @@ public abstract class MfSchemeDtoToEntityMapper {
void updateMFScheme(MFSchemeDTO scheme, @MappingTarget MFSchemeEntity mfSchemeEntity) {
MFSchemeNavEntity mfSchemenavEntity = new MFSchemeNavEntity();
mfSchemenavEntity.setNav("N.A.".equals(scheme.nav()) ? 0F : Float.parseFloat(scheme.nav()));
mfSchemenavEntity.setNavDate(LocalDate.parse(scheme.date(), AppConstants.FORMATTER_DD_MMM_YYYY));
LocalDate parsedDate;
if (scheme.date().length() == 10) {
parsedDate = LocalDate.parse(scheme.date());
} else {
parsedDate = LocalDate.parse(scheme.date(), AppConstants.FORMATTER_DD_MMM_YYYY);
}
mfSchemenavEntity.setNavDate(parsedDate);
mfSchemeEntity.addSchemeNav(mfSchemenavEntity);

MFSchemeTypeEntity mfSchemeTypeEntity = null;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.learning.mfscreener.repository;

import com.learning.mfscreener.entities.MFSchemeNavEntity;
import java.time.LocalDate;
import org.springframework.data.jpa.repository.JpaRepository;

public interface MFSchemeNavEntityRepository extends JpaRepository<MFSchemeNavEntity, Long> {

long countByNavDate(LocalDate navDate);
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,4 @@ where upper(m.fundHouse) like upper(:fName) order by m.schemeId
@Transactional(readOnly = true)
@Query("select o.schemeId from MFSchemeEntity o")
List<Long> findAllSchemeIds();

int countByMfSchemeNavEntities_NavDate(LocalDate grandFathertedDate);
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import com.learning.mfscreener.config.logging.Loggable;
import com.learning.mfscreener.entities.MFSchemeEntity;
import com.learning.mfscreener.entities.MFSchemeNavEntity;
import com.learning.mfscreener.exception.NavNotFoundException;
import com.learning.mfscreener.mapper.MfSchemeDtoToEntityMapper;
import com.learning.mfscreener.models.MFSchemeDTO;
Expand All @@ -16,8 +15,6 @@
import java.io.StringReader;
import java.net.URI;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import org.slf4j.Logger;
Expand All @@ -39,8 +36,6 @@ public class HistoricalNavService {
private final MfSchemeDtoToEntityMapper mfSchemeDtoToEntityMapper;
private final UserSchemeDetailsService userSchemeDetailsService;

List<Long> schemeCodeList = new ArrayList<>();

public HistoricalNavService(
SchemeService schemeService,
RestClient restClient,
Expand All @@ -64,12 +59,6 @@ public String getHistoricalNav(Long schemeCode, LocalDate navDate) {
}
}

public void getHistoricalNavOn31Jan2018() {
String toDate = AppConstants.GRAND_FATHERED_DATE.format(FORMATTER_DD_MMM_YYYY);
URI historicalNavUri = buildHistoricalNavUri(toDate, toDate);
fetchAndProcessNavData(historicalNavUri, null, true, null, AppConstants.GRAND_FATHERED_DATE);
}

String handleDiscontinuedScheme(Long schemeCode, URI historicalNavUri, LocalDate navDate) {
// discontinued scheme Isin
Optional<SchemeNameAndISIN> schemeNameAndISIN = userSchemeDetailsService.findFirstBySchemeCode(schemeCode);
Expand All @@ -96,16 +85,14 @@ String fetchAndProcessNavData(
String parseNavData(
Reader inputString, String isin, boolean persistSchemeInfo, Long schemeCode, LocalDate navDate) {
String oldSchemeId = null;
List<SchemeData> schemeDataList = new ArrayList<>();
try (BufferedReader br = new BufferedReader(inputString)) {
String lineValue = br.readLine();
for (int i = 0; i < 2; ++i) {
lineValue = br.readLine();
}
String schemeType = lineValue;
String amc = lineValue;
boolean insertEachRow = navDate.isEqual(AppConstants.GRAND_FATHERED_DATE);
while (lineValue != null && (insertEachRow || !StringUtils.hasText(oldSchemeId))) {
while (lineValue != null && !StringUtils.hasText(oldSchemeId)) {
String[] tokenize = lineValue.split(AppConstants.NAV_SEPARATOR);
if (tokenize.length == 1) {
String tempVal = lineValue;
Expand All @@ -117,52 +104,22 @@ String parseNavData(
} else {
amc = tempVal;
oldSchemeId = handleMultipleTokenLine(
isin,
persistSchemeInfo,
tokenize,
oldSchemeId,
amc,
schemeType,
schemeCode,
insertEachRow,
schemeDataList);
isin, persistSchemeInfo, tokenize, oldSchemeId, amc, schemeType, schemeCode);
}

} else {
oldSchemeId = handleMultipleTokenLine(
isin,
persistSchemeInfo,
tokenize,
oldSchemeId,
amc,
schemeType,
schemeCode,
insertEachRow,
schemeDataList);
isin, persistSchemeInfo, tokenize, oldSchemeId, amc, schemeType, schemeCode);
}
lineValue = readNextNonEmptyLine(br);
}
} catch (IOException e) {
LOGGER.error("Exception Occurred while reading response", e);
throw new NavNotFoundException("Unable to parse for %s".formatted(schemeCode), navDate);
}
if (!schemeDataList.isEmpty()) {
persistSchemeInfoBulk(schemeDataList);
}
return oldSchemeId;
}

void persistSchemeInfoBulk(List<SchemeData> schemeDataList) {
List<MFSchemeEntity> entitiesToPersist = new ArrayList<>();
for (SchemeData schemeData : schemeDataList) {
String[] tokenize = schemeData.tokenize();
MFSchemeEntity entity = createSchemeDTO(tokenize, schemeData.amc(), schemeData.schemeType());
entitiesToPersist.add(entity);
}
schemeService.saveAllEntities(entitiesToPersist);
LOGGER.debug("Persisted Entities: {}", entitiesToPersist.size());
}

String readNextNonEmptyLine(BufferedReader br) throws IOException {
String lineValue = br.readLine();
while (lineValue != null && !StringUtils.hasText(lineValue)) {
Expand All @@ -178,47 +135,23 @@ String handleMultipleTokenLine(
String oldSchemeId,
String amc,
String schemeType,
Long inputSchemeCode,
boolean insertEachRow,
List<SchemeData> schemeDataList) {
Long inputSchemeCode) {
final Long schemeCode = Long.valueOf(tokenize[0]);
final String payout = tokenize[2];
if (insertEachRow) {
schemeDataList.add(new SchemeData(tokenize, amc, schemeType));
} else if (payout.equalsIgnoreCase(isin) || schemeCode.equals(inputSchemeCode)) {
if (payout.equalsIgnoreCase(isin) || schemeCode.equals(inputSchemeCode)) {
oldSchemeId = String.valueOf(schemeCode);
if (persistSchemeInfo) {
schemeDataList.add(new SchemeData(tokenize, amc, schemeType));
String nav = tokenize[4];
String date = tokenize[7];
String schemeName = tokenize[1];
MFSchemeDTO mfSchemeDTO = new MFSchemeDTO(amc, schemeCode, payout, schemeName, nav, date, schemeType);
MFSchemeEntity mfSchemeEntity = mfSchemeDtoToEntityMapper.mapMFSchemeDTOToMFSchemeEntity(mfSchemeDTO);
schemeService.saveEntity(mfSchemeEntity);
}
}
return oldSchemeId;
}

MFSchemeEntity createSchemeDTO(String[] tokenize, String amc, String schemeType) {
List<Long> allSchemeIds = findAllSchemeIds();
Long schemeCode = Long.valueOf(tokenize[0]);
String payout = tokenize[2];
String nav = tokenize[4];
String date = tokenize[7];
if (allSchemeIds.contains(schemeCode)) {
MFSchemeNavEntity mfSchemenavEntity = new MFSchemeNavEntity();
mfSchemenavEntity.setNav("N.A.".equals(nav) ? 0F : Float.parseFloat(nav));
mfSchemenavEntity.setNavDate(LocalDate.parse(date, FORMATTER_DD_MMM_YYYY));
return schemeService.getReferenceBySchemeId(schemeCode).addSchemeNav(mfSchemenavEntity);
} else {
String schemeName = tokenize[1];
MFSchemeDTO mfSchemeDTO = new MFSchemeDTO(amc, schemeCode, payout, schemeName, nav, date, schemeType);
return mfSchemeDtoToEntityMapper.mapMFSchemeDTOToMFSchemeEntity(mfSchemeDTO);
}
}

List<Long> findAllSchemeIds() {
if (schemeCodeList.isEmpty()) {
schemeCodeList = schemeService.findAllSchemeIds();
}
return schemeCodeList;
}

String fetchHistoricalNavData(URI historicalNavUri) {
return restClient.get().uri(historicalNavUri).retrieve().body(String.class);
}
Expand All @@ -232,6 +165,4 @@ URI buildHistoricalNavUri(String toDate, String fromDate) {
.formatted(fromDate, toDate);
return UriComponentsBuilder.fromHttpUrl(historicalUrl).build().toUri();
}

public record SchemeData(String[] tokenize, String amc, String schemeType) {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package com.learning.mfscreener.service;

import com.learning.mfscreener.entities.MFSchemeNavEntity;
import com.learning.mfscreener.repository.MFSchemeNavEntityRepository;
import com.learning.mfscreener.repository.MFSchemeRepository;
import com.learning.mfscreener.utils.AppConstants;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

@Service
public class MFSchemeNavService {

private static final Logger LOGGER = LoggerFactory.getLogger(MFSchemeNavService.class);

private final MFSchemeNavEntityRepository mfSchemeNavEntityRepository;
private final MFSchemeRepository mfSchemeRepository;
private final ResourceLoader resourceLoader;

public MFSchemeNavService(
MFSchemeNavEntityRepository mfSchemeNavEntityRepository,
MFSchemeRepository mfSchemeRepository,
ResourceLoader resourceLoader) {
this.mfSchemeNavEntityRepository = mfSchemeNavEntityRepository;
this.mfSchemeRepository = mfSchemeRepository;
this.resourceLoader = resourceLoader;
}

public void loadHistoricalNavOn31Jan2018ForExistingSchemes() {

Resource resource = resourceLoader.getResource("classpath:/nav/31Jan2018Navdata.csv");
try {
Path path = resource.getFile().toPath();
List<String> lines = Files.lines(path).parallel().toList();
List<MFSchemeNavEntity> mfSchemeNavEntities = lines.stream()
.skip(1)
.map(csvRow -> {
// Split the row
String[] fields = csvRow.split(",");

// Trim and remove quotes
for (int i = 0; i < fields.length; i++) {
fields[i] = fields[i].trim().replaceAll("^\"+|\"+$", "");
}
MFSchemeNavEntity mfSchemeNavEntity = new MFSchemeNavEntity();
mfSchemeNavEntity.setNav(Float.valueOf(fields[0]));
mfSchemeNavEntity.setNavDate(AppConstants.GRAND_FATHERED_DATE);
mfSchemeNavEntity.setMfSchemeEntity(
mfSchemeRepository.getReferenceById(Long.valueOf(fields[2].replace("\"\"", ""))));
return mfSchemeNavEntity;
})
.toList();
List<MFSchemeNavEntity> persistedEntities = mfSchemeNavEntityRepository.saveAll(mfSchemeNavEntities);
LOGGER.info("Persisted : {} rows", persistedEntities.size());
} catch (IOException e) {
throw new RuntimeException(e);
}
}

@Transactional(readOnly = true)
public boolean navLoadedFor31Jan2018ForExistingSchemes() {
return mfSchemeNavEntityRepository.countByNavDate(AppConstants.GRAND_FATHERED_DATE) >= 5908;
}

@Transactional
public boolean navLoadedForClosedOrMergedSchemes() {
return mfSchemeNavEntityRepository.countByNavDate(AppConstants.GRAND_FATHERED_DATE) >= 9000;
}
}
Loading

0 comments on commit ac2c5da

Please sign in to comment.