fix: Wrong batch job status and error message (#2342)

tolgee · Jun 5, 2024 · 26ee281 · 26ee281
1 parent 9b08b7a
commit 26ee281
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 9 deletions.
diff --git a/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsCleanerTest.kt b/backend/app/src/test/kotlin/io/tolgee/batch/BatchJobsCleanerTest.kt
@@ -50,6 +50,7 @@ class BatchJobsCleanerTest : AbstractSpringTest() {
     val runningJob = createRunningJob()
     val pendingJob = createPendingJob()
     val dontTouchJob = createDontTouchJob()
+    val successWithRetriedChunks = createSuccessWithRetriedChunks()
 
     waitForNotThrowing(timeout = 2000, pollTime = 100) {
       batchJobService.getJobDto(cancelledJob.id).status.assert.isEqualTo(BatchJobStatus.CANCELLED)
@@ -59,6 +60,7 @@ class BatchJobsCleanerTest : AbstractSpringTest() {
       batchJobService.getJobDto(runningJob.id).status.assert.isEqualTo(BatchJobStatus.RUNNING)
       batchJobService.getJobDto(pendingJob.id).status.assert.isEqualTo(BatchJobStatus.PENDING)
       batchJobService.getJobDto(dontTouchJob.id).status.assert.isEqualTo(BatchJobStatus.FAILED)
+      batchJobService.getJobDto(successWithRetriedChunks.id).status.assert.isEqualTo(BatchJobStatus.SUCCESS)
     }
   }
 
@@ -119,4 +121,18 @@ class BatchJobsCleanerTest : AbstractSpringTest() {
         BatchJobChunkExecutionStatus.FAILED,
       ),
     )
+
+  private fun createSuccessWithRetriedChunks() =
+    stuckBatchJobTestUtil.createBatchJobWithExecutionStatuses(
+      testData.project,
+      BatchJobStatus.RUNNING,
+      mapOf(
+        1 to
+          listOf(
+            BatchJobChunkExecutionStatus.FAILED,
+            BatchJobChunkExecutionStatus.FAILED,
+            BatchJobChunkExecutionStatus.SUCCESS,
+          ),
+      ),
+    )
 }
diff --git a/backend/app/src/test/kotlin/io/tolgee/util/StuckBatchJobTestUtil.kt b/backend/app/src/test/kotlin/io/tolgee/util/StuckBatchJobTestUtil.kt
@@ -18,21 +18,40 @@ class StuckBatchJobTestUtil(
     project: Project,
     batchJobStatus: BatchJobStatus,
     executionStatuses: Set<BatchJobChunkExecutionStatus>,
+  ): BatchJob {
+    return createBatchJobWithExecutionStatuses(
+      project = project,
+      batchJobStatus = batchJobStatus,
+      executionStatuses =
+        executionStatuses.mapIndexed { index, status ->
+          index to listOf(status)
+        }.toMap(),
+    )
+  }
+
+  fun createBatchJobWithExecutionStatuses(
+    project: Project,
+    batchJobStatus: BatchJobStatus,
+    executionStatuses: Map<Int, List<BatchJobChunkExecutionStatus>>,
   ): BatchJob {
     return executeInNewTransaction(transactionManager) {
       val job =
         BatchJob().apply {
           status = batchJobStatus
           this.project = project
+          totalChunks = executionStatuses.size
         }
 
       entityManager.persist(job)
 
-      executionStatuses.map { status ->
-        BatchJobChunkExecution().apply {
-          batchJob = job
-          this.status = status
-          entityManager.persist(this)
+      executionStatuses.map { (chunkNumber, statuses) ->
+        statuses.map { status ->
+          BatchJobChunkExecution().apply {
+            batchJob = job
+            this.status = status
+            this.chunkNumber = chunkNumber
+            entityManager.persist(this)
+          }
         }
       }
       job

diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt b/backend/data/src/main/kotlin/io/tolgee/batch/BatchJobService.kt
@@ -274,7 +274,7 @@ class BatchJobService(
 
     return batchJobRepository.getErrorMessages(needsErrorMessage)
       .groupBy { it.batchJobId }
-      .mapValues { it.value.minBy { value -> value.updatedAt }.errorMessage }
+      .mapValues { it.value.maxBy { value -> value.updatedAt }.errorMessage }
   }
 
   private fun getProgresses(jobs: Iterable<BatchJob>): Map<Long, Int> {

diff --git a/backend/data/src/main/kotlin/io/tolgee/batch/cleaning/ScheduledJobCleaner.kt b/backend/data/src/main/kotlin/io/tolgee/batch/cleaning/ScheduledJobCleaner.kt
@@ -86,23 +86,33 @@ class ScheduledJobCleaner(
     val chunkIncompleteStatuses = BatchJobChunkExecutionStatus.entries.filter { !it.completed }.map { it.name }
     val jobIncompleteStatuses = BatchJobStatus.entries.filter { !it.completed }.map { it.name }
 
+    // In this query we are looking for jobs
+    // - that have incomplete status
+    // - have all chunks completed
+    // - and we are obtaining the resulting chunk status by preferably filtering out chunks which have success status
     val data =
       entityManager.createNativeQuery(
         """
         select tbj.id as id, tbj.project_id as projectId, tbjce2.status as status
         from tolgee_batch_job tbj
                  left join tolgee_batch_job_chunk_execution tbjce
                            on tbj.id = tbjce.batch_job_id and tbjce.status in :chunkIncompleteStatuses
-                 left join tolgee_batch_job_chunk_execution tbjce2 
+                 left join tolgee_batch_job_chunk_execution tbjce2
                            on tbj.id = tbjce2.batch_job_id
+                 left join tolgee_batch_job_chunk_execution tbjce_success
+                           on tbj.id = tbjce_success.batch_job_id 
+                             and tbjce_success.status = :successStatus 
+                             and tbjce2.chunk_number = tbjce_success.chunk_number  
+                             and tbjce_success.id <> tbjce2.id
         where tbj.status in :jobIncompleteStatuses
-          and tbjce.id is null
+          and tbjce.id is null and tbjce_success.id is null
         group by  tbj.id, tbj.project_id, tbjce2.status
       """,
         Array<Any>::class.java,
       )
         .setParameter("chunkIncompleteStatuses", chunkIncompleteStatuses)
         .setParameter("jobIncompleteStatuses", jobIncompleteStatuses)
+        .setParameter("successStatus", BatchJobChunkExecutionStatus.SUCCESS.name)
         .resultList as List<Array<Any>>
 
     return data.groupBy { it[0] }.map { rawDataList ->

diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/BatchJobRepository.kt
@@ -66,7 +66,8 @@ interface BatchJobRepository : JpaRepository<BatchJob, Long> {
      select bjce.batchJob.id as batchJobId, bjce.id as executionId, bjce.errorMessage as errorMessage, bjce.updatedAt as updatedAt 
      from BatchJobChunkExecution bjce 
        where bjce.batchJob.id in :jobIds 
-         and  bjce.errorMessage is not null
+         and  bjce.errorMessage is not null 
+         and bjce.retry = false
   """,
   )
   fun getErrorMessages(jobIds: List<Long>): List<JobErrorMessagesView>