fix(s3): Add front50-sql-mariadb to settings.gradle

front50-sql/src/main/kotlin/com/netflix/spinnaker/config/CompositeStorageServiceConfiguration.kt

@@ -20,6 +20,7 @@ import com.netflix.spectator.api.Registry
 import com.netflix.spinnaker.front50.migrations.StorageServiceMigrator
 import com.netflix.spinnaker.front50.model.CompositeStorageService
 import com.netflix.spinnaker.front50.model.StorageService
+import com.netflix.spinnaker.front50.model.tag.EntityTagsDAO
 import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
 import org.springframework.boot.context.properties.ConfigurationProperties
@@ -49,11 +50,13 @@ class CompositeStorageServiceConfiguration() {
   @ConditionalOnProperty("spinnaker.migration.enabled")
   fun storageServiceMigrator(registry: Registry,
                              properties: StorageServiceMigratorConfigurationProperties,
-                             storageServices: List<StorageService>) =
+                             storageServices: List<StorageService>,
+                             entityTagsDAO : EntityTagsDAO) =
     StorageServiceMigrator(
       registry,
       storageServices.first { it.javaClass.canonicalName.equals(properties.primaryClass) },
-      storageServices.first { it.javaClass.canonicalName.equals(properties.previousClass) }
+      storageServices.first { it.javaClass.canonicalName.equals(properties.previousClass) },
+      entityTagsDAO
     )
 }
 

front50-sql/src/main/kotlin/com/netflix/spinnaker/front50/migrations/StorageServiceMigrator.kt

@@ -20,6 +20,7 @@ import com.netflix.spectator.api.Registry
 import com.netflix.spinnaker.front50.model.ObjectType
 import com.netflix.spinnaker.front50.model.StorageService
 import com.netflix.spinnaker.front50.model.Timestamped
+import com.netflix.spinnaker.front50.model.tag.EntityTagsDAO
 import com.netflix.spinnaker.security.AuthenticatedRequest
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.async
@@ -34,7 +35,9 @@ import kotlin.system.measureTimeMillis
 class StorageServiceMigrator(
   private val registry: Registry,
   private val target: StorageService,
-  private val source: StorageService
+  private val source: StorageService,
+  private val entityTagsDAO: EntityTagsDAO
+
 ) {
 
   companion object {
@@ -46,7 +49,14 @@ class StorageServiceMigrator(
   fun migrate(objectType: ObjectType) {
     log.info("Migrating {}", objectType)
 
-    val sourceObjectKeys = source.listObjectKeys(objectType)
+    val sourceObjectKeys = if (objectType == ObjectType.ENTITY_TAGS) {
+      entityTagsDAO.all(false).map {
+        it.id to it.lastModified
+      }.toMap()
+    } else {
+      source.listObjectKeys(objectType)
+    }
+
     val targetObjectKeys = target.listObjectKeys(objectType)
 
     val deletableObjectKeys = targetObjectKeys.filter { e ->
@@ -178,4 +188,13 @@ class StorageServiceMigrator(
 
     log.info("Migration complete in {}ms", migrationDurationMs)
   }
+
+//  @Scheduled(fixedDelay = 90000)
+//  fun migrateEntityTags() {
+//    val migrationDurationMs = measureTimeMillis {
+//      migrate(ObjectType.ENTITY_TAGS)
+//    }
+//
+//    log.info("Entity Tags Migration complete in {}ms", migrationDurationMs)
+//  }
 }

front50-sql/src/main/kotlin/com/netflix/spinnaker/front50/model/CompositeStorageService.kt

@@ -104,6 +104,31 @@ class CompositeStorageService(
     }
   }
 
+  override fun <T : Timestamped?> loadObjects(objectType: ObjectType, objectKeys: List<String>): List<T> {
+    if (objectType == ObjectType.ENTITY_TAGS) {
+      // entity tags are currently unsupported in SQL
+      return previous.loadObjects<T>(objectType, objectKeys)
+    }
+
+    var exception: Exception? = null
+
+    if (isPrimaryReadEnabled()) {
+      try {
+        return primary.loadObjects<T>(objectType, objectKeys)
+      } catch (e: Exception) {
+        log.error("{}.loadObjects({}) failed (primary)", primary.javaClass.simpleName, objectType)
+
+        exception = e
+      }
+    }
+
+    return when {
+      isPreviousReadEnabled() -> previous.loadObjects<T>(objectType, objectKeys)
+      exception != null -> throw exception
+      else -> throw IllegalStateException("Primary and previous storage services are disabled")
+    }
+  }
+
   override fun deleteObject(objectType: ObjectType?, objectKey: String?) {
     primary.deleteObject(objectType, objectKey)
     previous.deleteObject(objectType, objectKey)
@@ -149,6 +174,10 @@ class CompositeStorageService(
   }
 
   override fun listObjectKeys(objectType: ObjectType?): Map<String, Long> {
+    if (objectType == ObjectType.ENTITY_TAGS) {
+      return previous.listObjectKeys(objectType)
+    }
+
     val objectKeys = mutableMapOf<String, Long>()
 
     if (isPreviousReadEnabled()) {

front50-sql/src/main/kotlin/com/netflix/spinnaker/front50/model/SqlStorageService.kt

@@ -30,7 +30,7 @@ import java.time.Clock
 import com.netflix.spinnaker.front50.model.ObjectType.*
 import com.netflix.spinnaker.front50.model.sql.*
 import com.netflix.spinnaker.kork.sql.config.SqlRetryProperties
-import java.util.ArrayList
+import kotlin.system.measureTimeMillis
 
 class SqlStorageService(
   private val objectMapper: ObjectMapper,
@@ -82,6 +82,41 @@ class SqlStorageService(
     return objectMapper.readValue(result.get(field("body", String::class.java)), objectType.clazz as Class<T>)
   }
 
+  override fun <T : Timestamped> loadObjects(objectType: ObjectType, objectKeys: List<String>): List<T> {
+    val objects = mutableListOf<T>()
+
+    val timeToLoadObjects = measureTimeMillis {
+      objects.addAll(
+        objectKeys.chunked(1000).flatMap { keys ->
+          val bodies = jooq.withRetry(sqlRetryProperties.reads) { ctx ->
+            ctx
+              .select(field("body", String::class.java))
+              .from(definitionsByType[objectType]!!.tableName)
+              .where(
+                field("id", String::class.java).`in`(keys).and(
+                  DSL.field("is_deleted", Boolean::class.java).eq(false)
+                )
+              )
+              .fetch()
+              .getValues(field("body", String::class.java))
+          }
+
+          bodies.map {
+            objectMapper.readValue(it, objectType.clazz as Class<T>)
+          }
+        }
+      )
+    }
+
+    log.debug("Took {}ms to fetch {} objects for {}",
+      timeToLoadObjects,
+      objects.size,
+      objectType
+    )
+
+    return objects
+  }
+
   override fun deleteObject(objectType: ObjectType, objectKey: String) {
     jooq.transactional(sqlRetryProperties.transactions) { ctx ->
       if (definitionsByType[objectType]!!.supportsHistory) {

front50-sql/src/test/kotlin/com/netflix/spinnaker/front50/model/SqlStorageServiceTests.kt

@@ -35,7 +35,7 @@ import java.time.Clock
 internal object SqlStorageServiceTests : JUnit5Minutests {
   private val jooq = initDatabase(
     "jdbc:tc:mysql:5.7.22://somehostname:someport/databasename",
-    SQLDialect.MYSQL_5_7
+    SQLDialect.MYSQL
   )
 
   private val sqlStorageService = SqlStorageService(