FINERACT-1724-COB-hardlocked-loans-and-inline-cob-issue

fineract-loan/src/main/java/org/apache/fineract/cob/exceptions/LoanAccountLockCannotBeOverruledException.java

@@ -23,4 +23,9 @@ public class LoanAccountLockCannotBeOverruledException extends RuntimeException
     public LoanAccountLockCannotBeOverruledException(String message) {
         super(message);
     }
+
+    public LoanAccountLockCannotBeOverruledException(String message, Exception e) {
+        super(message, e);
+    }
+
 }

fineract-provider/src/main/java/org/apache/fineract/cob/domain/LoanAccountLockRepository.java

@@ -55,4 +55,6 @@ lck.lockOwner in (org.apache.fineract.cob.domain.LockOwner.LOAN_COB_CHUNK_PROCES
             """)
     @Modifying(flushAutomatically = true)
     void removeLockByOwner();
+
+    List<LoanAccountLock> findAllByLoanIdInAndLockOwner(List<Long> loanIds, LockOwner lockOwner);
 }

fineract-provider/src/main/java/org/apache/fineract/cob/exceptions/LoanLockCannotBeAppliedException.java

@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.cob.exceptions;
+
+public class LoanLockCannotBeAppliedException extends Exception {
+
+    public LoanLockCannotBeAppliedException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java

@@ -18,6 +18,8 @@
  */
 package org.apache.fineract.cob.loan;
 
+import static org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW;
+
 import com.google.common.collect.Lists;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -29,26 +31,34 @@
 import org.apache.fineract.cob.data.LoanCOBParameter;
 import org.apache.fineract.cob.domain.LoanAccountLock;
 import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.cob.exceptions.LoanLockCannotBeAppliedException;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.jetbrains.annotations.NotNull;
 import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
 import org.springframework.batch.core.step.tasklet.Tasklet;
 import org.springframework.batch.item.ExecutionContext;
 import org.springframework.batch.repeat.RepeatStatus;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallbackWithoutResult;
+import org.springframework.transaction.support.TransactionTemplate;
 
 @Slf4j
 @RequiredArgsConstructor
 public class ApplyLoanLockTasklet implements Tasklet {
 
+    private static final long NUMBER_OF_RETRIES = 3;
     private final FineractProperties fineractProperties;
     private final LoanLockingService loanLockingService;
     private final RetrieveLoanIdService retrieveLoanIdService;
     private final CustomJobParameterResolver customJobParameterResolver;
+    private final TransactionTemplate transactionTemplate;
 
     @Override
-    public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception {
+    public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext)
+            throws LoanLockCannotBeAppliedException {
         ExecutionContext executionContext = contribution.getStepExecution().getExecutionContext();
+        long numberOfExecutions = contribution.getStepExecution().getCommitCount();
         LoanCOBParameter loanCOBParameter = (LoanCOBParameter) executionContext.get(LoanCOBConstant.LOAN_COB_PARAMETER);
         List<Long> loanIds;
         if (Objects.isNull(loanCOBParameter)
@@ -66,16 +76,36 @@ public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull Chu
         List<LoanAccountLock> accountLocks = new ArrayList<>();
         loanIdPartitions.forEach(loanIdPartition -> accountLocks.addAll(loanLockingService.findAllByLoanIdIn(loanIdPartition)));
 
-        List<Long> alreadyLockedByChunkProcessingAccountIds = accountLocks.stream()
-                .filter(e -> LockOwner.LOAN_COB_CHUNK_PROCESSING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList();
-
         List<Long> toBeProcessedLoanIds = new ArrayList<>(loanIds);
-        toBeProcessedLoanIds.removeAll(alreadyLockedByChunkProcessingAccountIds);
+        List<Long> alreadyLockedAccountIds = accountLocks.stream().map(LoanAccountLock::getLoanId).toList();
+
+        toBeProcessedLoanIds.removeAll(alreadyLockedAccountIds);
+        try {
+            applyLocks(toBeProcessedLoanIds);
+        } catch (Exception e) {
+            if (numberOfExecutions > NUMBER_OF_RETRIES) {
+                String message = "There was an error applying lock to loan accounts.";
+                log.error("{}", message, e);
+                throw new LoanLockCannotBeAppliedException(message, e);
+            } else {
+                return RepeatStatus.CONTINUABLE;
+            }
+        }
 
-        loanLockingService.applyLock(toBeProcessedLoanIds, LockOwner.LOAN_COB_CHUNK_PROCESSING);
         return RepeatStatus.FINISHED;
     }
 
+    private void applyLocks(List<Long> toBeProcessedLoanIds) {
+        transactionTemplate.setPropagationBehavior(PROPAGATION_REQUIRES_NEW);
+        transactionTemplate.execute(new TransactionCallbackWithoutResult() {
+
+            @Override
+            protected void doInTransactionWithoutResult(@NotNull TransactionStatus status) {
+                loanLockingService.applyLock(toBeProcessedLoanIds, LockOwner.LOAN_COB_CHUNK_PROCESSING);
+            }
+        });
+    }
+
     private int getInClauseParameterSizeLimit() {
         return fineractProperties.getQuery().getInClauseParameterSizeLimit();
     }

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java

@@ -131,7 +131,8 @@ public ChunkProcessingLoanItemListener loanItemListener() {
 
     @Bean
     public ApplyLoanLockTasklet applyLock() {
-        return new ApplyLoanLockTasklet(fineractProperties, loanLockingService, retrieveLoanIdService, customJobParameterResolver);
+        return new ApplyLoanLockTasklet(fineractProperties, loanLockingService, retrieveLoanIdService, customJobParameterResolver,
+                transactionTemplate);
     }
 
     @Bean
@@ -142,7 +143,7 @@ public ResetContextTasklet resetContext() {
     @Bean
     @StepScope
     public LoanItemReader cobWorkerItemReader() {
-        return new LoanItemReader(loanRepository, retrieveLoanIdService, customJobParameterResolver);
+        return new LoanItemReader(loanRepository, retrieveLoanIdService, customJobParameterResolver, loanLockingService);
     }
 
     @Bean

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanItemReader.java

@@ -24,6 +24,8 @@
 import java.util.Objects;
 import org.apache.fineract.cob.common.CustomJobParameterResolver;
 import org.apache.fineract.cob.data.LoanCOBParameter;
+import org.apache.fineract.cob.domain.LoanAccountLock;
+import org.apache.fineract.cob.domain.LockOwner;
 import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository;
 import org.jetbrains.annotations.NotNull;
 import org.springframework.batch.core.StepExecution;
@@ -34,12 +36,14 @@ public class LoanItemReader extends AbstractLoanItemReader {
 
     private final RetrieveLoanIdService retrieveLoanIdService;
     private final CustomJobParameterResolver customJobParameterResolver;
+    private final LoanLockingService loanLockingService;
 
     public LoanItemReader(LoanRepository loanRepository, RetrieveLoanIdService retrieveLoanIdService,
-            CustomJobParameterResolver customJobParameterResolver) {
+            CustomJobParameterResolver customJobParameterResolver, LoanLockingService loanLockingService) {
         super(loanRepository);
         this.retrieveLoanIdService = retrieveLoanIdService;
         this.customJobParameterResolver = customJobParameterResolver;
+        this.loanLockingService = loanLockingService;
     }
 
     @BeforeStep
@@ -56,7 +60,16 @@ public void beforeStep(@NotNull StepExecution stepExecution) {
             loanIds = retrieveLoanIdService.retrieveAllNonClosedLoansByLastClosedBusinessDateAndMinAndMaxLoanId(loanCOBParameter,
                     customJobParameterResolver.getCustomJobParameterById(stepExecution, LoanCOBConstant.IS_CATCH_UP_PARAMETER_NAME)
                             .map(Boolean::parseBoolean).orElse(false));
+
+            List<Long> lockedByCOBChunkProcessingAccountIds = getLoanIdsLockedWithChunkProcessingLock(loanIds);
+            loanIds.retainAll(lockedByCOBChunkProcessingAccountIds);
         }
         setRemainingData(new ArrayList<>(loanIds));
     }
+
+    private List<Long> getLoanIdsLockedWithChunkProcessingLock(List<Long> loanIds) {
+        List<LoanAccountLock> accountLocks = new ArrayList<>();
+        accountLocks.addAll(loanLockingService.findAllByLoanIdInAndLockOwner(loanIds, LockOwner.LOAN_COB_CHUNK_PROCESSING));
+        return accountLocks.stream().map(LoanAccountLock::getLoanId).toList();
+    }
 }

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanLockingService.java

@@ -32,5 +32,7 @@ public interface LoanLockingService {
 
     LoanAccountLock findByLoanIdAndLockOwner(Long loanId, LockOwner lockOwner);
 
+    List<LoanAccountLock> findAllByLoanIdInAndLockOwner(List<Long> loanIds, LockOwner lockOwner);
+
     void applyLock(List<Long> loanIds, LockOwner lockOwner);
 }

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanLockingServiceImpl.java

@@ -85,6 +85,11 @@ public LoanAccountLock findByLoanIdAndLockOwner(Long loanId, LockOwner lockOwner
         });
     }
 
+    @Override
+    public List<LoanAccountLock> findAllByLoanIdInAndLockOwner(List<Long> loanIds, LockOwner lockOwner) {
+        return loanAccountLockRepository.findAllByLoanIdInAndLockOwner(loanIds, lockOwner);
+    }
+
     @Override
     public void applyLock(List<Long> loanIds, LockOwner lockOwner) {
         LocalDate cobBusinessDate = ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE);

fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java

@@ -218,8 +218,14 @@ private void lockLoanAccounts(List<Long> loanIds, LocalDate businessDate) {
             protected void doInTransactionWithoutResult(@NotNull TransactionStatus status) {
                 List<LoanAccountLock> loanAccountLocks = getLoanAccountLocks(loanIds, businessDate);
                 loanAccountLocks.forEach(loanAccountLock -> {
-                    loanAccountLock.setNewLockOwner(LockOwner.LOAN_INLINE_COB_PROCESSING);
-                    loanAccountLockRepository.save(loanAccountLock);
+                    try {
+                        loanAccountLock.setNewLockOwner(LockOwner.LOAN_INLINE_COB_PROCESSING);
+                        loanAccountLockRepository.saveAndFlush(loanAccountLock);
+                    } catch (Exception e) {
+                        String message = "Error updating lock on loan account. Locked loan ID: %s".formatted(loanAccountLock.getLoanId());
+                        log.error("{}", message, e);
+                        throw new LoanAccountLockCannotBeOverruledException(message, e);
+                    }
                 });
             }
         });

fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java

@@ -23,6 +23,7 @@
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 
 import io.cucumber.java8.En;
@@ -35,6 +36,7 @@
 import org.apache.fineract.cob.data.LoanCOBParameter;
 import org.apache.fineract.cob.domain.LoanAccountLock;
 import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.cob.exceptions.LoanLockCannotBeAppliedException;
 import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
@@ -45,6 +47,8 @@
 import org.springframework.batch.core.StepExecution;
 import org.springframework.batch.item.ExecutionContext;
 import org.springframework.batch.repeat.RepeatStatus;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
 
 public class ApplyLoanLockTaskletStepDefinitions implements En {
 
@@ -54,10 +58,11 @@ public class ApplyLoanLockTaskletStepDefinitions implements En {
     private FineractProperties fineractProperties = mock(FineractProperties.class);
     private FineractProperties.FineractQueryProperties fineractQueryProperties = mock(FineractProperties.FineractQueryProperties.class);
     private RetrieveLoanIdService retrieveLoanIdService = mock(RetrieveLoanIdService.class);
+    private TransactionTemplate transactionTemplate = spy(TransactionTemplate.class);
 
     private CustomJobParameterResolver customJobParameterResolver = mock(CustomJobParameterResolver.class);
     private ApplyLoanLockTasklet applyLoanLockTasklet = new ApplyLoanLockTasklet(fineractProperties, loanLockingService,
-            retrieveLoanIdService, customJobParameterResolver);
+            retrieveLoanIdService, customJobParameterResolver, transactionTemplate);
     private RepeatStatus resultItem;
     private StepContribution stepContribution;
 
@@ -81,6 +86,25 @@ public ApplyLoanLockTaskletStepDefinitions() {
                 lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
                 lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
                 lenient().when(loanLockingService.findAllByLoanIdIn(Mockito.anyList())).thenThrow(new RuntimeException("fail"));
+            } else if ("db-error-first-try".equals(action)) {
+                LoanAccountLock lock1 = new LoanAccountLock(1L, LockOwner.LOAN_COB_CHUNK_PROCESSING, LocalDate.now(ZoneId.systemDefault()));
+                LoanAccountLock lock3 = new LoanAccountLock(3L, LockOwner.LOAN_INLINE_COB_PROCESSING,
+                        LocalDate.now(ZoneId.systemDefault()));
+                List<LoanAccountLock> accountLocks = List.of(lock1, lock3);
+                lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+                lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
+                lenient().when(loanLockingService.findAllByLoanIdIn(Mockito.anyList())).thenReturn(accountLocks);
+                Mockito.doThrow(new RuntimeException("db error")).when(loanLockingService).applyLock(Mockito.anyList(), any());
+            } else if ("db-error-not-recoverable".equals(action)) {
+                LoanAccountLock lock1 = new LoanAccountLock(1L, LockOwner.LOAN_COB_CHUNK_PROCESSING, LocalDate.now(ZoneId.systemDefault()));
+                LoanAccountLock lock3 = new LoanAccountLock(3L, LockOwner.LOAN_INLINE_COB_PROCESSING,
+                        LocalDate.now(ZoneId.systemDefault()));
+                List<LoanAccountLock> accountLocks = List.of(lock1, lock3);
+                stepContribution.getStepExecution().setCommitCount(4);
+                lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties);
+                lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
+                lenient().when(loanLockingService.findAllByLoanIdIn(Mockito.anyList())).thenReturn(accountLocks);
+                Mockito.doThrow(new RuntimeException("db error")).when(loanLockingService).applyLock(Mockito.anyList(), any());
             } else {
                 LoanAccountLock lock1 = new LoanAccountLock(1L, LockOwner.LOAN_COB_CHUNK_PROCESSING, LocalDate.now(ZoneId.systemDefault()));
                 LoanAccountLock lock3 = new LoanAccountLock(3L, LockOwner.LOAN_INLINE_COB_PROCESSING,
@@ -90,6 +114,7 @@ public ApplyLoanLockTaskletStepDefinitions() {
                 lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000);
                 lenient().when(loanLockingService.findAllByLoanIdIn(Mockito.anyList())).thenReturn(accountLocks);
             }
+            transactionTemplate.setTransactionManager(mock(PlatformTransactionManager.class));
             lenient().when(customJobParameterResolver.getCustomJobParameterSet(any())).thenReturn(Optional.empty());
 
         });
@@ -111,5 +136,15 @@ public ApplyLoanLockTaskletStepDefinitions() {
                 resultItem = applyLoanLockTasklet.execute(stepContribution, null);
             });
         });
+
+        Then("throw LoanLockCannotBeAppliedException exception ApplyLoanLockTasklet.execute method", () -> {
+            assertThrows(LoanLockCannotBeAppliedException.class, () -> {
+                resultItem = applyLoanLockTasklet.execute(stepContribution, null);
+            });
+        });
+
+        Then("ApplyLoanLockTasklet.execute result should be retry", () -> {
+            assertEquals(RepeatStatus.CONTINUABLE, resultItem);
+        });
     }
 }

fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanItemReaderStepDefinitions.java

@@ -33,8 +33,11 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import org.apache.fineract.cob.common.CustomJobParameterResolver;
 import org.apache.fineract.cob.data.LoanCOBParameter;
+import org.apache.fineract.cob.domain.LoanAccountLock;
+import org.apache.fineract.cob.domain.LockOwner;
 import org.apache.fineract.cob.exceptions.LoanReadException;
 import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
@@ -52,7 +55,10 @@ public class LoanItemReaderStepDefinitions implements En {
 
     private CustomJobParameterResolver customJobParameterResolver = mock(CustomJobParameterResolver.class);
 
-    private LoanItemReader loanItemReader = new LoanItemReader(loanRepository, retrieveLoanIdService, customJobParameterResolver);
+    private LoanLockingService lockingService = mock(LoanLockingService.class);
+
+    private LoanItemReader loanItemReader = new LoanItemReader(loanRepository, retrieveLoanIdService, customJobParameterResolver,
+            lockingService);
 
     private Loan loan = mock(Loan.class);
 
@@ -70,7 +76,7 @@ public LoanItemReaderStepDefinitions() {
             List<Long> splitAccounts = new ArrayList<>();
             if (!loanIds.isEmpty()) {
                 List<String> splitStr = Splitter.on(',').splitToList(loanIds);
-                splitAccounts = splitStr.stream().map(Long::parseLong).toList();
+                splitAccounts = splitStr.stream().map(Long::parseLong).collect(Collectors.toList());
                 minLoanId = splitAccounts.get(0);
                 maxLoanId = splitAccounts.get(splitAccounts.size() - 1);
             }
@@ -87,7 +93,16 @@ public LoanItemReaderStepDefinitions() {
             businessDates.put(BusinessDateType.BUSINESS_DATE, businessDate);
             businessDates.put(BusinessDateType.COB_DATE, businessDate.minusDays(1));
             ThreadLocalContextUtil.setBusinessDates(businessDates);
+            LoanAccountLock loanAccountLock = new LoanAccountLock(1L, LockOwner.LOAN_COB_CHUNK_PROCESSING, businessDate.minusDays(1));
+            LoanAccountLock loanAccountLockNegativeNumberTest = new LoanAccountLock(-1L, LockOwner.LOAN_COB_CHUNK_PROCESSING,
+                    businessDate.minusDays(1));
             lenient().when(customJobParameterResolver.getCustomJobParameterSet(any())).thenReturn(Optional.empty());
+            lenient().when(lockingService.findAllByLoanIdInAndLockOwner(List.of(1L), LockOwner.LOAN_COB_CHUNK_PROCESSING))
+                    .thenReturn(List.of(loanAccountLock));
+            lenient().when(lockingService.findAllByLoanIdInAndLockOwner(List.of(1L, 2L), LockOwner.LOAN_COB_CHUNK_PROCESSING))
+                    .thenReturn(List.of(loanAccountLock));
+            lenient().when(lockingService.findAllByLoanIdInAndLockOwner(List.of(-1L), LockOwner.LOAN_COB_CHUNK_PROCESSING))
+                    .thenReturn(List.of(loanAccountLockNegativeNumberTest));
 
             loanItemReader.beforeStep(stepExecution);