feat: Add project dashboard page #915 - migrate on startup

backend/app/build.gradle

@@ -118,6 +118,7 @@ dependencies {
     testImplementation("org.mockito.kotlin:mockito-kotlin:${findProperty("mockitoVersion")}")
     testCompile("io.findify:s3mock_2.13:0.2.6")
     testImplementation("io.socket:socket.io-client:1.0.1")
+    testImplementation group: 'org.springframework.batch', name: 'spring-batch-test', version: '4.3.5'
 
     /**
      * MISC

backend/app/src/main/kotlin/io/tolgee/MigrationJobsCommandLineRunner.kt

@@ -1,25 +1,18 @@
 package io.tolgee
 
-import io.tolgee.configuration.tolgee.TolgeeProperties
-import org.springframework.batch.core.Job
-import org.springframework.batch.core.JobParameters
-import org.springframework.batch.core.launch.JobLauncher
-import org.springframework.beans.factory.annotation.Qualifier
+import io.tolgee.jobs.migration.translationStats.TranslationsStatsUpdateJobRunner
 import org.springframework.boot.CommandLineRunner
 import org.springframework.context.ApplicationListener
 import org.springframework.context.event.ContextClosedEvent
 import org.springframework.stereotype.Component
 
 @Component
 class MigrationJobsCommandLineRunner(
-  val tolgeeProperties: TolgeeProperties,
-  @Qualifier("translationStatsJob")
-  val translationStatsJob: Job,
-  val jobLauncher: JobLauncher
-) :
-  CommandLineRunner, ApplicationListener<ContextClosedEvent> {
+  private val translationsStatsUpdateJobRunner: TranslationsStatsUpdateJobRunner
+) : CommandLineRunner, ApplicationListener<ContextClosedEvent> {
+
   override fun run(vararg args: String) {
-    jobLauncher.run(translationStatsJob, JobParameters())
+    translationsStatsUpdateJobRunner.run()
   }
 
   override fun onApplicationEvent(event: ContextClosedEvent) {

backend/app/src/test/kotlin/io/tolgee/jobs/migration/translationStats/TranslationStatsJobTest.kt

@@ -0,0 +1,102 @@
+package io.tolgee.jobs.migration.translationStats
+
+import io.tolgee.AbstractSpringTest
+import io.tolgee.development.testDataBuilder.data.TranslationsTestData
+import io.tolgee.repository.TranslationRepository
+import io.tolgee.testing.assertions.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.springframework.batch.core.BatchStatus
+import org.springframework.batch.core.Job
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.boot.test.context.SpringBootTest
+import org.springframework.test.context.transaction.TestTransaction
+import org.springframework.transaction.annotation.Transactional
+
+@SpringBootTest
+class TranslationStatsJobTest : AbstractSpringTest() {
+
+  @Autowired
+  @Qualifier("translationStatsJob")
+  lateinit var translationStatsJob: Job
+
+  @Autowired
+  lateinit var translationsStatsUpdateJobRunner: TranslationsStatsUpdateJobRunner
+
+  @Autowired
+  lateinit var translationRepository: TranslationRepository
+
+  @Test
+  @Transactional
+  fun `it adds the stats`() {
+    prepareData(10)
+
+    val instance = translationsStatsUpdateJobRunner.run()
+
+    assertStatsAdded()
+    assertThat(instance?.status).isEqualTo(BatchStatus.COMPLETED)
+  }
+
+  @Test
+  @Transactional
+  fun `it does not run multiple times for same `() {
+    prepareData()
+
+    // first - it really runs
+    val instance = translationsStatsUpdateJobRunner.run()
+    // nothing to migrate, no run
+    val instance2 = translationsStatsUpdateJobRunner.run()
+
+    assertThat(instance).isNotNull
+    assertThat(instance2).isNull()
+  }
+
+  @Test
+  @Transactional
+  fun `it runs again when new translation without stats is created`() {
+    val testData = prepareData()
+
+    val instance = translationsStatsUpdateJobRunner.run()
+
+    TestTransaction.start()
+    val newTranslationId = translationService.setForKey(testData.aKey, mapOf("en" to "Hellooooo!"))["en"]!!.id
+    entityManager
+      .createNativeQuery(
+        "update translation set word_count = null, character_count = null where id = $newTranslationId"
+      ).executeUpdate()
+    TestTransaction.flagForCommit()
+    TestTransaction.end()
+
+    val instance2 = translationsStatsUpdateJobRunner.run()
+    assertThat(instance2?.id).isNotEqualTo(instance?.id)
+  }
+
+  private fun assertStatsAdded() {
+    val translations = translationRepository.findAll().toMutableList()
+    translations.sortBy { it.id }
+
+    assertThat(translations)
+      .allMatch { it.wordCount != null }
+      .allMatch { it.characterCount != null }
+  }
+
+  private fun prepareData(keysToCreateCount: Long = 10): TranslationsTestData {
+    val testData = TranslationsTestData()
+    testData.generateLotOfData(keysToCreateCount)
+    testDataService.saveTestData(testData.root)
+
+    commitTransaction()
+
+    entityManager
+      .createNativeQuery("update translation set word_count = null, character_count = null")
+      .executeUpdate()
+
+    commitTransaction()
+
+    val translations = translationRepository.findAll()
+    assertThat(translations).allMatch { it.wordCount == null }.allMatch { it.characterCount == null }
+
+    TestTransaction.end()
+    return testData
+  }
+}

backend/data/src/main/kotlin/io/tolgee/jobs/migration/translationStats/StatsMigrationTranslationView.kt

@@ -0,0 +1,7 @@
+package io.tolgee.jobs.migration.translationStats
+
+interface StatsMigrationTranslationView {
+  val id: Long
+  val text: String?
+  val languageTag: String
+}

backend/data/src/main/kotlin/io/tolgee/jobs/migration/translationStats/TranslationProcessor.kt

@@ -1,13 +1,13 @@
 package io.tolgee.jobs.migration.translationStats
 
-import io.tolgee.model.translation.Translation
 import io.tolgee.util.WordCounter
 import org.springframework.batch.item.ItemProcessor
 
-class TranslationProcessor : ItemProcessor<Translation, Translation> {
-  override fun process(item: Translation): Translation {
-    item.wordCount = item.text?.let { WordCounter.countWords(it, item.language.tag) }
-    item.characterCount = item.text?.length ?: 0
-    return item
+class TranslationProcessor : ItemProcessor<StatsMigrationTranslationView, TranslationStats> {
+  override fun process(item: StatsMigrationTranslationView): TranslationStats {
+    return TranslationStats(
+      id = item.id, wordCount = item.text?.let { WordCounter.countWords(it, item.languageTag) } ?: 0,
+      characterCount = item.text?.length ?: 0
+    )
   }
 }

backend/data/src/main/kotlin/io/tolgee/jobs/migration/translationStats/TranslationStats.kt

@@ -0,0 +1,7 @@
+package io.tolgee.jobs.migration.translationStats
+
+class TranslationStats(
+  val id: Long,
+  val wordCount: Int,
+  val characterCount: Int
+)

backend/data/src/main/kotlin/io/tolgee/jobs/migration/translationStats/TranslationStatsJobConfiguration.kt

@@ -1,21 +1,27 @@
 package io.tolgee.jobs.migration.translationStats
 
-import io.tolgee.model.translation.Translation
 import io.tolgee.repository.TranslationRepository
 import org.springframework.batch.core.Job
 import org.springframework.batch.core.Step
 import org.springframework.batch.core.configuration.annotation.JobBuilderFactory
 import org.springframework.batch.core.configuration.annotation.StepBuilderFactory
+import org.springframework.batch.item.ItemReader
+import org.springframework.batch.item.ItemWriter
 import org.springframework.batch.item.data.RepositoryItemReader
-import org.springframework.batch.item.data.RepositoryItemWriter
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.data.domain.Sort
 import javax.persistence.EntityManager
+import javax.sql.DataSource
 
 @Configuration
 class TranslationStatsJobConfiguration {
+
+  companion object {
+    const val JOB_NAME = "translationStatsJob"
+  }
+
   @Autowired
   lateinit var jobBuilderFactory: JobBuilderFactory
 
@@ -28,31 +34,40 @@ class TranslationStatsJobConfiguration {
   @Autowired
   lateinit var translationRepository: TranslationRepository
 
-  @Bean("translationStatsJob")
+  @Autowired
+  lateinit var dataSource: DataSource
+
+  @Bean(JOB_NAME)
   fun translationStatsJob(): Job {
     return jobBuilderFactory["translationStats"]
       .flow(step)
       .end()
       .build()
   }
 
-  val reader: RepositoryItemReader<Translation>
-    get() = RepositoryItemReader<Translation>().apply {
+  val reader: ItemReader<StatsMigrationTranslationView>
+    get() = RepositoryItemReader<StatsMigrationTranslationView>().apply {
       setRepository(translationRepository)
-      setMethodName("findAll")
+      setMethodName(translationRepository::findAllForStatsUpdate.name)
       setSort(mapOf("id" to Sort.Direction.ASC))
-      setPageSize(10)
+      setPageSize(100)
     }
 
-  val writer: RepositoryItemWriter<Translation>
-    get() = RepositoryItemWriter<Translation>().apply {
-      setRepository(translationRepository)
-      setMethodName("save")
+  val writer: ItemWriter<TranslationStats> = ItemWriter { items ->
+    items.forEach {
+      val query = entityManager.createNativeQuery(
+        "UPDATE translation set word_count = :wordCount, character_count = :characterCount where id = :id"
+      )
+      query.setParameter("wordCount", it.wordCount)
+      query.setParameter("characterCount", it.characterCount)
+      query.setParameter("id", it.id)
+      query.executeUpdate()
     }
+  }
 
   val step: Step
     get() = stepBuilderFactory["step"]
-      .chunk<Translation, Translation>(100)
+      .chunk<StatsMigrationTranslationView, TranslationStats>(100)
       .reader(reader)
       .processor(TranslationProcessor())
       .writer(writer)

backend/data/src/main/kotlin/io/tolgee/jobs/migration/translationStats/TranslationsStatsUpdateJobRunner.kt

@@ -0,0 +1,46 @@
+package io.tolgee.jobs.migration.translationStats
+
+import io.tolgee.configuration.tolgee.TolgeeProperties
+import io.tolgee.repository.TranslationRepository
+import org.apache.commons.codec.digest.DigestUtils
+import org.slf4j.LoggerFactory
+import org.springframework.batch.core.Job
+import org.springframework.batch.core.JobExecution
+import org.springframework.batch.core.JobParameter
+import org.springframework.batch.core.JobParameters
+import org.springframework.batch.core.launch.JobLauncher
+import org.springframework.batch.core.repository.JobRepository
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.stereotype.Component
+
+@Component
+class TranslationsStatsUpdateJobRunner(
+  val tolgeeProperties: TolgeeProperties,
+  @Qualifier(TranslationStatsJobConfiguration.JOB_NAME)
+  val translationStatsJob: Job,
+  val jobLauncher: JobLauncher,
+  val translationRepository: TranslationRepository,
+  val jobRepository: JobRepository
+) {
+
+  val log = LoggerFactory.getLogger(this::class.java)
+
+  fun run(): JobExecution? {
+    val params = getJobParams()
+
+    if (params != null) {
+      return jobRepository.getLastJobExecution(TranslationStatsJobConfiguration.JOB_NAME, params)
+        ?: return jobLauncher.run(translationStatsJob, params)
+    }
+    return null
+  }
+
+  private fun getJobParams(): JobParameters? {
+    val ids = translationRepository.findAllIdsForStatsUpdate()
+    if (ids.isEmpty()) {
+      return null
+    }
+    val hash = DigestUtils.sha256Hex(ids.flatMap { it.toBigInteger().toByteArray().toList() }.toByteArray())
+    return JobParameters(mapOf("idsHash" to JobParameter(hash)))
+  }
+}

backend/data/src/main/kotlin/io/tolgee/model/key/Key.kt

@@ -12,6 +12,7 @@ import io.tolgee.model.dataImport.WithKeyMeta
 import io.tolgee.model.translation.Translation
 import javax.persistence.Column
 import javax.persistence.Entity
+import javax.persistence.FetchType
 import javax.persistence.ManyToOne
 import javax.persistence.OneToMany
 import javax.persistence.OneToOne
@@ -41,7 +42,7 @@ class Key(
   @OneToMany(mappedBy = "key")
   var translations: MutableSet<Translation> = HashSet()
 
-  @OneToOne(mappedBy = "key")
+  @OneToOne(mappedBy = "key", fetch = FetchType.LAZY)
   override var keyMeta: KeyMeta? = null
 
   @OneToMany(mappedBy = "key")

backend/data/src/main/kotlin/io/tolgee/model/translation/Translation.kt

@@ -13,6 +13,7 @@ import org.hibernate.annotations.ColumnDefault
 import javax.persistence.Column
 import javax.persistence.Entity
 import javax.persistence.Enumerated
+import javax.persistence.FetchType
 import javax.persistence.ManyToOne
 import javax.persistence.OneToMany
 import javax.persistence.Table
@@ -37,12 +38,11 @@ class Translation(
   @ActivityDescribingProp
   var text: String? = null
 ) : StandardAuditModel() {
-
-  @ManyToOne(optional = false)
+  @ManyToOne(optional = false, fetch = FetchType.LAZY)
   @field:NotNull
   lateinit var key: Key
 
-  @ManyToOne
+  @ManyToOne(fetch = FetchType.LAZY)
   lateinit var language: Language
 
   @Enumerated

backend/data/src/main/kotlin/io/tolgee/repository/TranslationRepository.kt

@@ -1,5 +1,6 @@
 package io.tolgee.repository
 
+import io.tolgee.jobs.migration.translationStats.StatsMigrationTranslationView
 import io.tolgee.model.Language
 import io.tolgee.model.Project
 import io.tolgee.model.key.Key
@@ -100,4 +101,22 @@ interface TranslationRepository : JpaRepository<Translation, Long> {
     targetLanguage: Language,
     pageable: Pageable = PageRequest.of(0, 1)
   ): List<TranslationMemoryItemView>
+
+  @Query(
+    """
+    select t.id as id, t.text as text, t.language.tag as languageTag
+    from Translation t
+  """
+  )
+  fun findAllForStatsUpdate(pageable: Pageable): Page<StatsMigrationTranslationView>
+
+  @Query(
+    """
+    select t.id
+    from Translation t
+    where t.text <> null and (t.wordCount is null or t.characterCount is null or (length(text) <> 0 and t.characterCount = 0))
+    order by t.id
+  """
+  )
+  fun findAllIdsForStatsUpdate(): List<Long>
 }

backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt

@@ -1,7 +1,6 @@
 package io.tolgee.testing
 
 import io.tolgee.CleanDbTestListener
-import io.tolgee.repository.LanguageRepository
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.test.context.TestExecutionListeners
 import org.springframework.test.context.support.DependencyInjectionTestExecutionListener
@@ -20,9 +19,6 @@ abstract class AbstractTransactionalTest {
   @Autowired
   protected lateinit var entityManager: EntityManager
 
-  @Autowired
-  protected lateinit var languageRepository: LanguageRepository
-
   protected fun commitTransaction() {
     TestTransaction.flagForCommit()
     entityManager.flush()