diff --git a/front50-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt b/front50-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt index 8abe99898..6557f254d 100644 --- a/front50-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt +++ b/front50-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt @@ -39,5 +39,12 @@ class SqlConfiguration : CommonStorageServiceDAOConfig() { registry: Registry, jooq: DSLContext, sqlProperties: SqlProperties): SqlStorageService = - SqlStorageService(objectMapper, registry, jooq, Clock.systemDefaultZone(), sqlProperties.retries) + SqlStorageService( + objectMapper, + registry, + jooq, + Clock.systemDefaultZone(), + sqlProperties.retries, + 1000 + ) } diff --git a/front50-sql/src/main/kotlin/com/netflix/spinnaker/front50/model/SqlStorageService.kt b/front50-sql/src/main/kotlin/com/netflix/spinnaker/front50/model/SqlStorageService.kt index fad8ea353..79e03997e 100644 --- a/front50-sql/src/main/kotlin/com/netflix/spinnaker/front50/model/SqlStorageService.kt +++ b/front50-sql/src/main/kotlin/com/netflix/spinnaker/front50/model/SqlStorageService.kt @@ -37,7 +37,8 @@ class SqlStorageService( private val registry: Registry, private val jooq: DSLContext, private val clock: Clock, - private val sqlRetryProperties: SqlRetryProperties + private val sqlRetryProperties: SqlRetryProperties, + private val chunkSize: Int ) : StorageService { companion object { @@ -87,7 +88,7 @@ class SqlStorageService( val timeToLoadObjects = measureTimeMillis { objects.addAll( - objectKeys.chunked(1000).flatMap { keys -> + objectKeys.chunked(chunkSize).flatMap { keys -> val bodies = jooq.withRetry(sqlRetryProperties.reads) { ctx -> ctx .select(field("body", String::class.java)) diff --git a/front50-sql/src/test/kotlin/com/netflix/spinnaker/front50/model/SqlStorageServiceTests.kt b/front50-sql/src/test/kotlin/com/netflix/spinnaker/front50/model/SqlStorageServiceTests.kt index 27ec240b2..c208a5ca9 100644 --- a/front50-sql/src/test/kotlin/com/netflix/spinnaker/front50/model/SqlStorageServiceTests.kt +++ b/front50-sql/src/test/kotlin/com/netflix/spinnaker/front50/model/SqlStorageServiceTests.kt @@ -43,7 +43,8 @@ internal object SqlStorageServiceTests : JUnit5Minutests { NoopRegistry(), jooq, Clock.systemDefaultZone(), - SqlRetryProperties() + SqlRetryProperties(), + 1 ) fun tests() = rootContext { @@ -143,6 +144,34 @@ internal object SqlStorageServiceTests : JUnit5Minutests { sqlStorageService.loadObject(ObjectType.PIPELINE, "id-pipeline001") } } + + test("bulk load pipelines") { + val objectKeys = mutableSetOf() + (1..10).forEach { + val objectKey = "id-pipeline00$it" + objectKeys.add(objectKey) + + sqlStorageService.storeObject( + ObjectType.PIPELINE, + objectKey, + Pipeline().apply { + id = objectKey + name = "pipeline00$it" + lastModified = 100 + + put("application", "application001") + } + ) + } + + val pipelines = sqlStorageService.loadObjects( + ObjectType.PIPELINE, + (objectKeys + "does_not_exist").toList() + ) + expectThat( + pipelines.map { it.id }.toSet() + ).isEqualTo(objectKeys) + } } context("Entity Tags") {