From a0f7b18edfbd222e02032e372f278c8f233bfda4 Mon Sep 17 00:00:00 2001 From: Asher Feldman Date: Mon, 1 Jul 2019 15:59:31 -0700 Subject: [PATCH] chore(sql): convert tests to use tc-mysql, unpin jooq (#3832) --- cats/cats-sql/cats-sql.gradle | 7 ++- .../spinnaker/cats/sql/cache/SqlCache.kt | 50 ++++++------------- .../spinnaker/cats/sql/SqlCacheSpec.groovy | 21 +++++--- .../cats/sql/SqlProviderCacheSpec.groovy | 17 +++++-- .../spinnaker/cats/cache/CacheSpec.groovy | 2 +- .../ops/discovery/AwsEurekaSupport.groovy | 2 - clouddriver-sql/clouddriver-sql.gradle | 7 ++- .../sql/SqlTaskRepositoryTest.java | 8 +-- .../ops/discovery/TitusEurekaSupport.groovy | 2 - gradle.properties | 4 +- 10 files changed, 50 insertions(+), 70 deletions(-) diff --git a/cats/cats-sql/cats-sql.gradle b/cats/cats-sql/cats-sql.gradle index 1af3e148e7e..b1ec306c212 100644 --- a/cats/cats-sql/cats-sql.gradle +++ b/cats/cats-sql/cats-sql.gradle @@ -32,7 +32,6 @@ dependencies { testAnnotationProcessor "org.projectlombok:lombok" implementation "com.fasterxml.jackson.core:jackson-databind" - implementation "com.h2database:h2:1.4.197" implementation "com.netflix.spectator:spectator-api" implementation "com.netflix.spinnaker.fiat:fiat-api:$fiatVersion" implementation "com.netflix.spinnaker.fiat:fiat-core:$fiatVersion" @@ -46,9 +45,7 @@ dependencies { implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core-common:1.1.1" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.1.1" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-slf4j:1.1.1" - implementation("org.jooq:jooq:3.9.6"){ - force = true - } + implementation("org.jooq:jooq") implementation "org.springframework.boot:spring-boot-starter-web" testImplementation project(":cats:cats-test") @@ -67,4 +64,6 @@ dependencies { testImplementation "org.spockframework:spock-core" testImplementation "org.spockframework:spock-spring" testImplementation "org.springframework:spring-test" + testImplementation "org.testcontainers:mysql" + testImplementation "mysql:mysql-connector-java" } diff --git a/cats/cats-sql/src/main/kotlin/com/netflix/spinnaker/cats/sql/cache/SqlCache.kt b/cats/cats-sql/src/main/kotlin/com/netflix/spinnaker/cats/sql/cache/SqlCache.kt index d7931956ab1..5676d2298e5 100644 --- a/cats/cats-sql/src/main/kotlin/com/netflix/spinnaker/cats/sql/cache/SqlCache.kt +++ b/cats/cats-sql/src/main/kotlin/com/netflix/spinnaker/cats/sql/cache/SqlCache.kt @@ -841,25 +841,14 @@ class SqlCache( private fun createTables(type: String) { if (!createdTables.contains(type)) { try { - if (jooq.dialect().name != "H2") { - withRetry(RetryCategory.WRITE) { - jooq.execute("CREATE TABLE IF NOT EXISTS ${resourceTableName(type)} " + - "LIKE cats_v${schemaVersion}_resource_template") - jooq.execute("CREATE TABLE IF NOT EXISTS ${relTableName(type)} " + - "LIKE cats_v${schemaVersion}_rel_template") - } - - createdTables.add(type) - } else { - withRetry(RetryCategory.WRITE) { - jooq.execute("CREATE TABLE ${resourceTableName(type)} " + - "AS SELECT * FROM cats_v${schemaVersion}_resource_template WHERE 1=0") - jooq.execute("CREATE TABLE ${relTableName(type)} " + - "AS SELECT * FROM cats_v${schemaVersion}_rel_template WHERE 1=0") - } - - createdTables.add(type) + withRetry(RetryCategory.WRITE) { + jooq.execute("CREATE TABLE IF NOT EXISTS ${resourceTableName(type)} " + + "LIKE cats_v${schemaVersion}_resource_template") + jooq.execute("CREATE TABLE IF NOT EXISTS ${relTableName(type)} " + + "LIKE cats_v${schemaVersion}_rel_template") } + + createdTables.add(type) } catch (e: Exception) { log.error("Error creating tables for type $type", e) } @@ -867,25 +856,14 @@ class SqlCache( if (!createdTables.contains(onDemandType)) { // TODO not sure if best schema for onDemand try { - if (jooq.dialect().name != "H2") { - withRetry(RetryCategory.WRITE) { - jooq.execute("CREATE TABLE IF NOT EXISTS ${resourceTableName(onDemandType)} " + - "LIKE cats_v${schemaVersion}_resource_template") - jooq.execute("CREATE TABLE IF NOT EXISTS ${relTableName(onDemandType)} " + - "LIKE cats_v${schemaVersion}_rel_template") - } - - createdTables.add(onDemandType) - } else { - withRetry(RetryCategory.WRITE) { - jooq.execute("CREATE TABLE ${resourceTableName(onDemandType)} " + - "AS SELECT * FROM cats_v${schemaVersion}_resource_template WHERE 1=0") - jooq.execute("CREATE TABLE ${relTableName(onDemandType)} " + - "AS SELECT * FROM cats_v${schemaVersion}_rel_template WHERE 1=0") - } - - createdTables.add(onDemandType) + withRetry(RetryCategory.WRITE) { + jooq.execute("CREATE TABLE IF NOT EXISTS ${resourceTableName(onDemandType)} " + + "LIKE cats_v${schemaVersion}_resource_template") + jooq.execute("CREATE TABLE IF NOT EXISTS ${relTableName(onDemandType)} " + + "LIKE cats_v${schemaVersion}_rel_template") } + + createdTables.add(onDemandType) } catch (e: Exception) { log.error("Error creating $onDemandType table", e) } diff --git a/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlCacheSpec.groovy b/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlCacheSpec.groovy index 6f0305f090a..5f865dafb7b 100644 --- a/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlCacheSpec.groovy +++ b/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlCacheSpec.groovy @@ -9,6 +9,9 @@ import com.netflix.spinnaker.cats.sql.cache.SqlCacheMetrics import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService import com.netflix.spinnaker.kork.sql.config.RetryProperties import com.netflix.spinnaker.kork.sql.config.SqlRetryProperties +import com.netflix.spinnaker.kork.sql.test.SqlTestUtil +import com.zaxxer.hikari.HikariDataSource +import org.jooq.DSLContext import spock.lang.AutoCleanup import spock.lang.Shared import spock.lang.Unroll @@ -17,17 +20,16 @@ import java.time.Clock import java.time.Instant import java.time.ZoneId -import static com.netflix.spinnaker.kork.sql.test.SqlTestUtil.TestDatabase -import static com.netflix.spinnaker.kork.sql.test.SqlTestUtil.initDatabase - class SqlCacheSpec extends WriteableCacheSpec { @Shared + DSLContext context + @AutoCleanup("close") - TestDatabase currentDatabase + HikariDataSource dataSource def cleanup() { - currentDatabase.context.dropSchemaIfExists("test") + SqlTestUtil.cleanupDb(context) } def 'should not write an item if it is unchanged'() { @@ -38,7 +40,7 @@ class SqlCacheSpec extends WriteableCacheSpec { ((SqlCache) cache).merge('foo', data) then: - 1 * ((SqlCache) cache).cacheMetrics.merge('test', 'foo', 1, 1, 0, 0, 2, 1, 0) + 1 * ((SqlCache) cache).cacheMetrics.merge('test', 'foo', 1, 1, 0, 0, 1, 1, 0) when: ((SqlCache) cache).merge('foo', data) @@ -91,10 +93,13 @@ class SqlCacheSpec extends WriteableCacheSpec { getConfig(_ as Class, _ as String, _) >> 2 } - currentDatabase = initDatabase("jdbc:h2:mem:test${System.currentTimeMillis()}") + SqlTestUtil.TestDatabase testDatabase = SqlTestUtil.initTcMysqlDatabase() + context = testDatabase.context + dataSource = testDatabase.dataSource + return new SqlCache( "test", - currentDatabase.context, + context, mapper, null, clock, diff --git a/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlProviderCacheSpec.groovy b/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlProviderCacheSpec.groovy index baa8d54f185..df6a3207630 100644 --- a/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlProviderCacheSpec.groovy +++ b/cats/cats-sql/src/test/groovy/com/netflix/spinnaker/cats/sql/SqlProviderCacheSpec.groovy @@ -14,6 +14,8 @@ import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService import com.netflix.spinnaker.kork.sql.config.RetryProperties import com.netflix.spinnaker.kork.sql.config.SqlRetryProperties import com.netflix.spinnaker.kork.sql.test.SqlTestUtil +import com.zaxxer.hikari.HikariDataSource +import org.jooq.DSLContext import spock.lang.AutoCleanup import spock.lang.Shared import spock.lang.Unroll @@ -22,18 +24,19 @@ import java.time.Clock import java.time.Instant import java.time.ZoneId -import static com.netflix.spinnaker.kork.sql.test.SqlTestUtil.initDatabase class SqlProviderCacheSpec extends ProviderCacheSpec { @Shared + DSLContext context + @AutoCleanup("close") - SqlTestUtil.TestDatabase currentDatabase + HikariDataSource dataSource WriteableCache backingStore def cleanup() { - currentDatabase.context.dropSchemaIfExists("test") + SqlTestUtil.cleanupDb(context) } @Override @@ -50,10 +53,14 @@ class SqlProviderCacheSpec extends ProviderCacheSpec { def dynamicConfigService = Mock(DynamicConfigService) { getConfig(_ as Class, _ as String, _) >> 10 } - currentDatabase = initDatabase("jdbc:h2:mem:test${System.currentTimeMillis()}") + + SqlTestUtil.TestDatabase testDatabase = SqlTestUtil.initTcMysqlDatabase() + context = testDatabase.context + dataSource = testDatabase.dataSource + backingStore = new SqlCache( "test", - currentDatabase.context, + context, mapper, null, clock, diff --git a/cats/cats-test/src/main/groovy/com/netflix/spinnaker/cats/cache/CacheSpec.groovy b/cats/cats-test/src/main/groovy/com/netflix/spinnaker/cats/cache/CacheSpec.groovy index 03b180b82b1..7ec9ad1ee84 100644 --- a/cats/cats-test/src/main/groovy/com/netflix/spinnaker/cats/cache/CacheSpec.groovy +++ b/cats/cats-test/src/main/groovy/com/netflix/spinnaker/cats/cache/CacheSpec.groovy @@ -95,7 +95,7 @@ abstract class CacheSpec extends Specification { '*TEST*' | ['blaTEST', 'blaTESTbla', 'TESTbla'] 'bla*' | ['blaTEST', 'blaTESTbla', 'blaPest', 'blaFEST'] 'bla[TF]EST' | ['blaTEST', 'blaFEST'] - 'bla?EST' | ['blaTEST', 'blaFEST'] + 'bla????' | ['blaTEST', 'blaPest', 'blaFEST'] '??a[FTP][Ee][Ss][Tt]*' | ['blaTEST', 'blaTESTbla', 'blaPest', 'blaFEST'] identifiers = ['blaTEST', 'TESTbla', 'blaTESTbla', 'blaPest', 'blaFEST'] diff --git a/clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/discovery/AwsEurekaSupport.groovy b/clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/discovery/AwsEurekaSupport.groovy index e8aac1ff012..29ee150b742 100644 --- a/clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/discovery/AwsEurekaSupport.groovy +++ b/clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/deploy/ops/discovery/AwsEurekaSupport.groovy @@ -45,8 +45,6 @@ class AwsEurekaSupport extends AbstractEurekaSupport { return eureka } - @VisibleForTesting - @PackageScope boolean verifyInstanceAndAsgExist(def credentials, String region, String instanceId, diff --git a/clouddriver-sql/clouddriver-sql.gradle b/clouddriver-sql/clouddriver-sql.gradle index 7f41b416931..e89bb582c82 100644 --- a/clouddriver-sql/clouddriver-sql.gradle +++ b/clouddriver-sql/clouddriver-sql.gradle @@ -25,12 +25,11 @@ dependencies { implementation "com.netflix.spinnaker.kork:kork-sql" implementation "de.huxhorn.sulky:de.huxhorn.sulky.ulid" implementation "io.github.resilience4j:resilience4j-retry" - implementation("org.jooq:jooq:3.9.6") { - force = true - - } + implementation "org.jooq:jooq" testImplementation project(":clouddriver-core-tck") testImplementation "com.netflix.spinnaker.kork:kork-sql-test" + testImplementation "org.testcontainers:mysql" + testImplementation "mysql:mysql-connector-java" } diff --git a/clouddriver-sql/src/test/java/com/netflix/spinnaker/clouddriver/sql/SqlTaskRepositoryTest.java b/clouddriver-sql/src/test/java/com/netflix/spinnaker/clouddriver/sql/SqlTaskRepositoryTest.java index 9b77a756622..1b0be066c09 100644 --- a/clouddriver-sql/src/test/java/com/netflix/spinnaker/clouddriver/sql/SqlTaskRepositoryTest.java +++ b/clouddriver-sql/src/test/java/com/netflix/spinnaker/clouddriver/sql/SqlTaskRepositoryTest.java @@ -22,8 +22,6 @@ import com.netflix.spinnaker.kork.sql.config.SqlRetryProperties; import com.netflix.spinnaker.kork.sql.test.SqlTestUtil; import java.time.Clock; -import java.util.Arrays; -import java.util.Optional; import org.junit.After; public class SqlTaskRepositoryTest extends TaskRepositoryTck { @@ -32,7 +30,7 @@ public class SqlTaskRepositoryTest extends TaskRepositoryTck { @Override protected TaskRepository createTaskRepository() { - database = SqlTestUtil.initDatabase("jdbc:h2:mem:test" + System.currentTimeMillis()); + database = SqlTestUtil.initTcMysqlDatabase(); RetryProperties retry = new RetryProperties(0, 0); SqlRetryProperties properties = @@ -46,8 +44,6 @@ protected TaskRepository createTaskRepository() { @After public void cleanup() { - Optional.ofNullable(database) - .ifPresent( - d -> SqlTestUtil.cleanupDb(d, Arrays.asList("tasks", "task_states", "task_results"))); + SqlTestUtil.cleanupDb(database.context); } } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/ops/discovery/TitusEurekaSupport.groovy b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/ops/discovery/TitusEurekaSupport.groovy index 445b4afe623..0bf36921480 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/ops/discovery/TitusEurekaSupport.groovy +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/ops/discovery/TitusEurekaSupport.groovy @@ -42,8 +42,6 @@ class TitusEurekaSupport extends AbstractEurekaSupport { EurekaUtil.getWritableEureka(credentials.discovery, region) } - @VisibleForTesting - @PackageScope boolean verifyInstanceAndAsgExist(def credentials, String region, String instanceId, diff --git a/gradle.properties b/gradle.properties index 749ff732fd2..b728c58770b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ -#Mon Jun 17 22:30:32 UTC 2019 +#Mon Jul 1 20:15:07 UTC 2019 fiatVersion=1.1.0 includeCloudProviders=all enablePublishing=false spinnakerGradleVersion=6.5.0 -korkVersion=5.5.2 +korkVersion=5.8.0 org.gradle.parallel=true