From d6b74636815a82a6eb5385a7ff19ead21cce1f7d Mon Sep 17 00:00:00 2001 From: Rob Zienert Date: Wed, 4 Sep 2019 14:32:10 -0700 Subject: [PATCH] refactor(titus): Allow serde of operation descriptions (#4001) --- clouddriver-core/clouddriver-core.gradle | 2 + .../config/CloudDriverConfig.groovy | 6 +- .../event/AbstractSpinnakerEvent.kt | 35 ++ .../spinnaker/clouddriver/event/Aggregate.kt | 2 + .../event/CompositeSpinnakerEvent.kt | 31 ++ .../clouddriver/event/EventMetadata.kt | 6 + .../clouddriver/event/SpinnakerEvent.kt | 18 +- .../config/MemoryEventRepositoryConfig.kt | 4 +- .../AggregateChangeRejectedException.kt | 8 +- .../event/exceptions/EventingException.kt | 21 + .../exceptions/InvalidEventTypeException.kt | 23 ++ .../exceptions/UninitializedEventException.kt | 26 ++ .../event/persistence/EventRepository.kt | 30 +- .../persistence/InMemoryEventRepository.kt | 51 ++- .../clouddriver/event/ObjectMappingTest.kt | 71 ++++ .../InMemoryEventRepositoryTest.kt | 38 +- .../clouddriver/saga/TestingSagaRepository.kt | 12 +- .../spinnaker/clouddriver/saga/types.kt | 1 + .../spinnaker/clouddriver/saga/SagaService.kt | 40 +- .../saga/config/SagaAutoConfiguration.kt | 11 + .../spinnaker/clouddriver/saga/events.kt | 15 +- .../spinnaker/clouddriver/saga/models/Saga.kt | 2 +- .../saga/persistence/DefaultSagaRepository.kt | 25 +- .../saga/persistence/SagaRepository.kt | 2 + .../titus/client/RegionScopedTitusClient.java | 6 +- .../titus/client/model/JobDescription.java | 8 +- .../titus/client/model/SubmitJobRequest.java | 389 ++---------------- .../actions/AbstractTitusDeployAction.java | 10 + .../AttachTitusServiceLoadBalancers.java | 50 ++- .../CopyTitusServiceScalingPolicies.java | 42 +- .../titus/deploy/actions/LoadFront50App.java | 42 +- .../deploy/actions/PrepareTitusDeploy.java | 55 ++- .../titus/deploy/actions/SubmitTitusJob.java | 120 +++--- .../deploy/actions/TitusJobNameResolver.java | 10 +- .../actions/TitusServiceJobPredicate.java | 7 +- .../AbstractTitusCredentialsDescription.java | 27 +- .../description/TitusDeployDescription.java | 95 +++-- .../deploy/events/TitusJobSubmitted.java | 24 +- .../events/TitusLoadBalancerAttached.java | 20 +- .../events/TitusScalingPolicyCopied.java | 23 +- .../deploy/handlers/TitusDeployHandler.java | 10 +- .../config/TitusConfiguration.groovy | 11 + .../TitusDeployDescriptionSpec.groovy | 79 ++++ .../handlers/actions/CommandSerdeSpec.groovy | 124 ++++++ .../PrepareTitusDeployActionSpec.groovy | 8 +- .../actions/TitusJobNameResolverSpec.groovy | 7 +- 46 files changed, 953 insertions(+), 694 deletions(-) create mode 100644 clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/AbstractSpinnakerEvent.kt create mode 100644 clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/CompositeSpinnakerEvent.kt create mode 100644 clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/EventingException.kt create mode 100644 clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/InvalidEventTypeException.kt create mode 100644 clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/UninitializedEventException.kt create mode 100644 clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/ObjectMappingTest.kt create mode 100644 clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/descriptions/TitusDeployDescriptionSpec.groovy create mode 100644 clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/CommandSerdeSpec.groovy diff --git a/clouddriver-core/clouddriver-core.gradle b/clouddriver-core/clouddriver-core.gradle index 5e938e094bc..2e75a920058 100644 --- a/clouddriver-core/clouddriver-core.gradle +++ b/clouddriver-core/clouddriver-core.gradle @@ -8,6 +8,8 @@ dependencies { testAnnotationProcessor "org.projectlombok:lombok" implementation "net.logstash.logback:logstash-logback-encoder" + implementation "com.fasterxml.jackson.module:jackson-module-kotlin" + implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" implementation "com.netflix.eureka:eureka-client" implementation "com.netflix.frigga:frigga" implementation "com.netflix.spinnaker.fiat:fiat-api:$fiatVersion" diff --git a/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.groovy b/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.groovy index e6b4bc9ff82..9264887dc26 100644 --- a/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.groovy +++ b/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.groovy @@ -18,17 +18,18 @@ package com.netflix.spinnaker.clouddriver.config import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.KotlinModule import com.netflix.spectator.api.Registry import com.netflix.spinnaker.cats.agent.Agent import com.netflix.spinnaker.cats.agent.ExecutionInstrumentation import com.netflix.spinnaker.cats.agent.NoopExecutionInstrumentation import com.netflix.spinnaker.cats.redis.cache.RedisCacheOptions import com.netflix.spinnaker.clouddriver.cache.CacheConfig -import com.netflix.spinnaker.clouddriver.cache.CustomScheduledAgent import com.netflix.spinnaker.clouddriver.cache.NoopOnDemandCacheUpdater import com.netflix.spinnaker.clouddriver.cache.OnDemandCacheUpdater import com.netflix.spinnaker.clouddriver.core.CloudProvider - import com.netflix.spinnaker.clouddriver.core.NoopAtomicOperationConverter import com.netflix.spinnaker.clouddriver.core.NoopCloudProvider import com.netflix.spinnaker.clouddriver.core.ProjectClustersService @@ -127,6 +128,7 @@ class CloudDriverConfig { jacksonObjectMapperBuilder.serializationInclusion(JsonInclude.Include.NON_NULL) jacksonObjectMapperBuilder.failOnEmptyBeans(false) jacksonObjectMapperBuilder.failOnUnknownProperties(false) + jacksonObjectMapperBuilder.modules(new Jdk8Module(), new JavaTimeModule(), new KotlinModule()) } } } diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/AbstractSpinnakerEvent.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/AbstractSpinnakerEvent.kt new file mode 100644 index 00000000000..8d4858fb2b1 --- /dev/null +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/AbstractSpinnakerEvent.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.event + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.netflix.spinnaker.clouddriver.event.exceptions.UninitializedEventException + +abstract class AbstractSpinnakerEvent : SpinnakerEvent { + /** + * Not a lateinit to make Java/Lombok & Jackson compatibility a little easier, although behavior is exactly the same. + */ + @JsonIgnore + private var metadata: EventMetadata? = null + + override fun getMetadata(): EventMetadata { + return metadata ?: throw UninitializedEventException() + } + + override fun setMetadata(eventMetadata: EventMetadata) { + metadata = eventMetadata + } +} diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/Aggregate.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/Aggregate.kt index 2cc2af27956..63100735e60 100644 --- a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/Aggregate.kt +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/Aggregate.kt @@ -23,6 +23,8 @@ package com.netflix.spinnaker.clouddriver.event * latest event state; any modification to an [Aggregate] event log will increment this value. * When an operation is attempted on an [version] which is not head, the event framework will * reject the change. + * + * TODO(rz): Add `currentSequence` to make resuming aggregate processing in-flight easier. */ class Aggregate( val type: String, diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/CompositeSpinnakerEvent.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/CompositeSpinnakerEvent.kt new file mode 100644 index 00000000000..23b8930e02f --- /dev/null +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/CompositeSpinnakerEvent.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.event + +import com.fasterxml.jackson.annotation.JsonIgnore + +/** + * Marks a [SpinnakerEvent] as being constructed of multiple [SpinnakerEvent]s. + * + * This interface is necessary to correctly hydrate [EventMetadata] on [SpinnakerEvent] before persisting. + */ +interface CompositeSpinnakerEvent : SpinnakerEvent { + /** + * Returns a list of the composed [SpinnakerEvent]s. + */ + @JsonIgnore + fun getComposedEvents(): List +} diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/EventMetadata.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/EventMetadata.kt index 3788e45010b..3ac4f625019 100644 --- a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/EventMetadata.kt +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/EventMetadata.kt @@ -20,6 +20,9 @@ import java.time.Instant /** * Metadata for a [SpinnakerEvent]. * + * @param id A unique ID for the event (not used beyond tracing, debugging) + * @param aggregateType The type of aggregate the event is for + * @param aggregateId The id of the aggregate the event is for * @param sequence Auto-incrementing number for event ordering * @param originatingVersion The aggregate version that originated this event * @param timestamp The time at which the event was created @@ -27,6 +30,9 @@ import java.time.Instant * @param source Where/what generated the event */ data class EventMetadata( + val id: String, + val aggregateType: String, + val aggregateId: String, val sequence: Long, val originatingVersion: Long, val timestamp: Instant = Instant.now(), diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/SpinnakerEvent.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/SpinnakerEvent.kt index ef8a8e38c95..6b520a0c02f 100644 --- a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/SpinnakerEvent.kt +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/SpinnakerEvent.kt @@ -16,25 +16,17 @@ package com.netflix.spinnaker.clouddriver.event import com.fasterxml.jackson.annotation.JsonTypeInfo -import java.util.UUID /** - * The base event class for the event sourcing library. - * - * @property id A unique ID for the event. This value is for tracing, rather than loading events - * @property aggregateType The type of aggregate the event is for - * @property aggregateId The id of the aggregate the event is for - * @property metadata Associated metadata about the event; not actually part of the "event proper" + * The base type for the eventing library. All library-level code is contained within [EventMetadata]. */ @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, - property = "spinEventType" + property = "eventType" ) -abstract class SpinnakerEvent { - val id = UUID.randomUUID().toString() +interface SpinnakerEvent { + fun getMetadata(): EventMetadata - lateinit var aggregateType: String - lateinit var aggregateId: String - lateinit var metadata: EventMetadata + fun setMetadata(eventMetadata: EventMetadata) } diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/config/MemoryEventRepositoryConfig.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/config/MemoryEventRepositoryConfig.kt index 3ffa4f3e024..fcb120f621b 100644 --- a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/config/MemoryEventRepositoryConfig.kt +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/config/MemoryEventRepositoryConfig.kt @@ -19,7 +19,7 @@ import com.netflix.spectator.api.Registry import com.netflix.spinnaker.clouddriver.event.persistence.EventRepository import com.netflix.spinnaker.clouddriver.event.persistence.InMemoryEventRepository import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.ApplicationEventPublisher @@ -34,7 +34,7 @@ import javax.validation.constraints.Min import kotlin.reflect.KClass @Configuration -@ConditionalOnProperty("spinnaker.clouddriver.eventing.memory-repository.enabled", matchIfMissing = true) +@ConditionalOnMissingBean(EventRepository::class) @EnableConfigurationProperties(MemoryEventRepositoryConfigProperties::class) open class MemoryEventRepositoryConfig { diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/AggregateChangeRejectedException.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/AggregateChangeRejectedException.kt index e6ed61b6fea..95b1c4b2b91 100644 --- a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/AggregateChangeRejectedException.kt +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/AggregateChangeRejectedException.kt @@ -22,6 +22,12 @@ import com.netflix.spinnaker.kork.exceptions.SystemException * * The process which originated the event must be retryable. */ -class AggregateChangeRejectedException(message: String) : SystemException(message) { +class AggregateChangeRejectedException( + aggregateVersion: Long, + originatingVersion: Long +) : SystemException( + "Attempting to save new events against an old aggregate version " + + "(version: $aggregateVersion, originatingVersion: $originatingVersion)" +) { override fun getRetryable() = true } diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/EventingException.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/EventingException.kt new file mode 100644 index 00000000000..71280cb1ca1 --- /dev/null +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/EventingException.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.event.exceptions + +/** + * Marker + */ +interface EventingException diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/InvalidEventTypeException.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/InvalidEventTypeException.kt new file mode 100644 index 00000000000..f3a862500ed --- /dev/null +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/InvalidEventTypeException.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.event.exceptions + +import com.netflix.spinnaker.kork.exceptions.IntegrationException + +/** + * Thrown when a [SpinnakerEvent] cannot be created. + */ +class InvalidEventTypeException(cause: Throwable) : IntegrationException(cause), EventingException diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/UninitializedEventException.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/UninitializedEventException.kt new file mode 100644 index 00000000000..5ca5a759bab --- /dev/null +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/exceptions/UninitializedEventException.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.event.exceptions + +import com.netflix.spinnaker.kork.exceptions.IntegrationException + +/** + * Thrown when an event's metadata is attempted to be retrieved before it has been initialized + * by the library. + */ +class UninitializedEventException : IntegrationException( + "Cannot access event metadata before initialization" +), EventingException diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/EventRepository.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/EventRepository.kt index 1b157d60017..81858403a07 100644 --- a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/EventRepository.kt +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/EventRepository.kt @@ -17,6 +17,8 @@ package com.netflix.spinnaker.clouddriver.event.persistence import com.netflix.spinnaker.clouddriver.event.Aggregate import com.netflix.spinnaker.clouddriver.event.SpinnakerEvent +import javax.validation.constraints.Max +import javax.validation.constraints.Positive /** * The [EventRepository] is responsible for reading and writing immutable event logs from a persistent store. @@ -49,8 +51,30 @@ interface EventRepository { /** * List all aggregates for a given type. * - * @param aggregateType The aggregate collection name. If not provided, all aggregates will be returned. - * @return An unordered list of matching aggregates. + * @param criteria The criteria to limit the response by + * @return A list of matching aggregates */ - fun listAggregates(aggregateType: String?): List + fun listAggregates(criteria: ListAggregatesCriteria): ListAggregatesResult + + /** + * @param aggregateType The type of [Aggregate] to return. If unset, all types will be returned. + * @param token The page token to paginate from. It will return the first results + * @param perPage The number of [Aggregate]s to return in each response + */ + class ListAggregatesCriteria( + val aggregateType: String? = null, + val token: String? = null, + + @Positive @Max(1000) + val perPage: Int = 100 + ) + + /** + * @param aggregates The collection of [Aggregate]s returned + * @param nextPageToken The next page token + */ + class ListAggregatesResult( + val aggregates: List, + val nextPageToken: String? = null + ) } diff --git a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepository.kt b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepository.kt index aa5a8cc6a61..c02ac899acb 100644 --- a/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepository.kt +++ b/clouddriver-event/src/main/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepository.kt @@ -27,6 +27,7 @@ import org.springframework.context.ApplicationEventPublisher import org.springframework.scheduling.annotation.Scheduled import java.time.Duration import java.time.Instant +import java.util.UUID import java.util.concurrent.ConcurrentHashMap import kotlin.math.max @@ -63,23 +64,21 @@ class InMemoryEventRepository( if (aggregate.version != originatingVersion) { // If this is being thrown, ensure that the originating process is retried on the latest aggregate version // by re-reading the newEvents list. - throw AggregateChangeRejectedException( - "Attempting to save newEvents against an old aggregate version " + - "(version: ${aggregate.version}, originatingVersion: $originatingVersion)") + throw AggregateChangeRejectedException(aggregate.version, originatingVersion) } events.getOrPut(aggregate) { mutableListOf() }.let { aggregateEvents -> - val currentSequence = aggregateEvents.map { it.metadata.sequence }.max() ?: 0 + val currentSequence = aggregateEvents.map { it.getMetadata().sequence }.max() ?: 0 newEvents.forEachIndexed { index, newEvent -> - newEvent.aggregateType = aggregateType - newEvent.aggregateId = aggregateId - // TODO(rz): Plugin more metadata (provenance, serviceVersion, etc) - newEvent.metadata = EventMetadata( + newEvent.setMetadata(EventMetadata( + id = UUID.randomUUID().toString(), + aggregateType = aggregateType, + aggregateId = aggregateId, sequence = currentSequence + (index + 1), originatingVersion = originatingVersion - ) + )) } registry.counter(eventWriteCountId).increment(newEvents.size.toLong()) @@ -103,13 +102,31 @@ class InMemoryEventRepository( ?: throw MissingAggregateEventsException(aggregateType, aggregateId) } - override fun listAggregates(aggregateType: String?): List { + override fun listAggregates(criteria: EventRepository.ListAggregatesCriteria): EventRepository.ListAggregatesResult { val aggregates = events.keys - return if (aggregateType != null) { - aggregates.filter { it.type == aggregateType } - } else { - aggregates.toList() - } + + val result = aggregates.toList() + .let { list -> + criteria.aggregateType?.let { requiredType -> list.filter { it.type == requiredType } } ?: list + } + .let { list -> + criteria.token?.let { nextPageToken -> + val start = list.indexOf(list.find { "${it.type}/${it.id}" == nextPageToken }) + val end = (start + criteria.perPage).let { + if (it > list.size - 1) { + list.size + } else { + criteria.perPage + } + } + list.subList(start, end) + } ?: list + } + + return EventRepository.ListAggregatesResult( + aggregates = result, + nextPageToken = result.lastOrNull()?.let { "${it.type}/${it.id}" } + ) } private fun getAggregate(aggregateType: String, aggregateId: String): Aggregate { @@ -134,7 +151,7 @@ class InMemoryEventRepository( val horizon = Instant.now().minus(maxAge) log.info("Cleaning up aggregates last updated earlier than $maxAge ($horizon)") events.entries - .filter { it.value.any { event -> event.metadata.timestamp.isBefore(horizon) } } + .filter { it.value.any { event -> event.getMetadata().timestamp.isBefore(horizon) } } .map { it.key } .forEach { log.trace("Cleaning up $it") @@ -150,7 +167,7 @@ class InMemoryEventRepository( .flatMap { entry -> entry.value.map { Pair(entry.key, it) } } - .sortedBy { it.second.metadata.timestamp } + .sortedBy { it.second.getMetadata().timestamp } .subList(0, max(events.size - maxCount, 0)) .forEach { log.trace("Cleaning up ${it.first}") diff --git a/clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/ObjectMappingTest.kt b/clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/ObjectMappingTest.kt new file mode 100644 index 00000000000..0c34b455693 --- /dev/null +++ b/clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/ObjectMappingTest.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.event + +import com.fasterxml.jackson.annotation.JsonTypeName +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import dev.minutest.junit.JUnit5Minutests +import dev.minutest.rootContext +import strikt.api.expectThat +import strikt.assertions.isA +import strikt.assertions.isEqualTo + +class ObjectMappingTest : JUnit5Minutests { + + fun tests() = rootContext { + fixture { + ObjectMapper() + .registerKotlinModule() + .findAndRegisterModules() + .apply { + registerSubtypes(listOf(MyEvent::class.java)) + } + } + + test("can serialize and deserialize events") { + val event = MyEvent("world") + event.setMetadata(EventMetadata( + id = "myid", + aggregateType = "type", + aggregateId = "id", + sequence = 999, + originatingVersion = 100 + )) + + val serialized = writeValueAsString(event) + + expectThat(readValue(serialized)) + .isA() + .and { + get { hello }.isEqualTo("world") + get { getMetadata() }.and { + get { id }.isEqualTo("myid") + get { aggregateType }.isEqualTo("type") + get { aggregateId }.isEqualTo("id") + get { sequence }.isEqualTo(999) + get { originatingVersion }.isEqualTo(100) + } + } + } + } + + @JsonTypeName("myEvent") + class MyEvent( + val hello: String + ) : AbstractSpinnakerEvent() +} diff --git a/clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepositoryTest.kt b/clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepositoryTest.kt index da3cb6398e9..85c377ff275 100644 --- a/clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepositoryTest.kt +++ b/clouddriver-event/src/test/kotlin/com/netflix/spinnaker/clouddriver/event/persistence/InMemoryEventRepositoryTest.kt @@ -16,9 +16,10 @@ package com.netflix.spinnaker.clouddriver.event.persistence import com.netflix.spectator.api.NoopRegistry -import com.netflix.spinnaker.clouddriver.event.SpinnakerEvent +import com.netflix.spinnaker.clouddriver.event.AbstractSpinnakerEvent import com.netflix.spinnaker.clouddriver.event.config.MemoryEventRepositoryConfigProperties import com.netflix.spinnaker.clouddriver.event.exceptions.AggregateChangeRejectedException +import com.netflix.spinnaker.clouddriver.event.persistence.EventRepository.ListAggregatesCriteria import dev.minutest.junit.JUnit5Minutests import dev.minutest.rootContext import io.mockk.confirmVerified @@ -48,7 +49,7 @@ class InMemoryEventRepositoryTest : JUnit5Minutests { test("save appends aggregate events") { val event = MyEvent("agg", "id", "hello world") - subject.save(event.aggregateType, event.aggregateId, 0L, listOf(event)) + subject.save("agg", "id", 0L, listOf(event)) expectThat(subject.list("agg", "id")) { get { size }.isEqualTo(1) @@ -58,7 +59,7 @@ class InMemoryEventRepositoryTest : JUnit5Minutests { } val event2 = MyEvent("agg", "id", "hello rob") - subject.save(event2.aggregateType, event2.aggregateId, 1L, listOf(event2)) + subject.save("agg", "id", 1L, listOf(event2)) expectThat(subject.list("agg", "id")) { get { size }.isEqualTo(2) @@ -74,22 +75,22 @@ class InMemoryEventRepositoryTest : JUnit5Minutests { test("saving with a new aggregate with a non-zero originating version fails") { val event = MyEvent("agg", "id", "hello") assertThrows { - subject.save(event.aggregateType, event.aggregateId, 10L, listOf(event)) + subject.save("agg", "id", 10L, listOf(event)) } } test("saving an aggregate with an old originating version fails") { val event = MyEvent("agg", "id", "hello") - subject.save(event.aggregateType, event.aggregateId, 0L, listOf(event)) + subject.save("agg", "id", 0L, listOf(event)) assertThrows { - subject.save(event.aggregateType, event.aggregateId, 0L, listOf(event)) + subject.save("agg", "id", 0L, listOf(event)) } } test("newly saved events are published") { val event = MyEvent("agg", "id", "hello") - subject.save(event.aggregateType, event.aggregateId, 0L, listOf(event)) + subject.save("agg", "id", 0L, listOf(event)) verify { eventPublisher.publishEvent(event) } confirmVerified(eventPublisher) @@ -105,8 +106,8 @@ class InMemoryEventRepositoryTest : JUnit5Minutests { subject.save(it.aggregateType, it.aggregateId, 0L, listOf(it)) } - expectThat(subject.listAggregates(null)) { - map { it.type }.containsExactly("type1", "type2", "type3") + expectThat(subject.listAggregates(ListAggregatesCriteria())) { + get { aggregates }.map { it.type }.containsExactly("type1", "type2", "type3") } } @@ -115,8 +116,8 @@ class InMemoryEventRepositoryTest : JUnit5Minutests { subject.save(it.aggregateType, it.aggregateId, 0L, listOf(it)) } - expectThat(subject.listAggregates(event1.aggregateType)) { - map { it.type }.containsExactly("type1") + expectThat(subject.listAggregates(ListAggregatesCriteria(aggregateType = event1.getMetadata().aggregateType))) { + get { aggregates }.map { it.type }.containsExactly("type1") } } @@ -125,8 +126,8 @@ class InMemoryEventRepositoryTest : JUnit5Minutests { subject.save(it.aggregateType, it.aggregateId, 0L, listOf(it)) } - expectThat(subject.listAggregates("unknown")) { - isEmpty() + expectThat(subject.listAggregates(ListAggregatesCriteria(aggregateType = "unknown"))) { + get { aggregates }.isEmpty() } } } @@ -142,13 +143,8 @@ class InMemoryEventRepositoryTest : JUnit5Minutests { } private inner class MyEvent( - aggregateType: String, - aggregateId: String, + val aggregateType: String, + val aggregateId: String, val value: String - ) : SpinnakerEvent() { - init { - this.aggregateType = aggregateType - this.aggregateId = aggregateId - } - } + ) : AbstractSpinnakerEvent() } diff --git a/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/TestingSagaRepository.kt b/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/TestingSagaRepository.kt index 35e619b6dc4..43f4eaa329c 100644 --- a/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/TestingSagaRepository.kt +++ b/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/TestingSagaRepository.kt @@ -18,6 +18,7 @@ package com.netflix.spinnaker.clouddriver.saga import com.netflix.spinnaker.clouddriver.event.EventMetadata import com.netflix.spinnaker.clouddriver.saga.models.Saga import com.netflix.spinnaker.clouddriver.saga.persistence.SagaRepository +import java.util.UUID class TestingSagaRepository : SagaRepository { @@ -34,18 +35,19 @@ class TestingSagaRepository : SagaRepository { override fun save(saga: Saga, additionalEvents: List) { sagas.putIfAbsent(createId(saga), saga) - val currentSequence = saga.getEvents().map { it.metadata.sequence }.max() ?: 0 + val currentSequence = saga.getEvents().map { it.getMetadata().sequence }.max() ?: 0 val originatingVersion = saga.getVersion() saga.getPendingEvents() .plus(additionalEvents) .forEachIndexed { index, event -> - event.aggregateType = saga.name - event.aggregateId = saga.id - event.metadata = EventMetadata( + event.setMetadata(EventMetadata( + id = UUID.randomUUID().toString(), + aggregateType = saga.name, + aggregateId = saga.id, sequence = currentSequence + index + 1, originatingVersion = originatingVersion - ) + )) saga.addEventForTest(event) } } diff --git a/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/types.kt b/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/types.kt index 98265c8bd64..c4246b59e44 100644 --- a/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/types.kt +++ b/clouddriver-saga-test/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/types.kt @@ -20,6 +20,7 @@ import com.netflix.spinnaker.clouddriver.saga.flow.SagaAction import com.netflix.spinnaker.clouddriver.saga.models.Saga import java.util.function.Predicate +@JsonTypeName("shouldBranch") class ShouldBranch : SagaEvent() @JsonTypeName("doAction1") diff --git a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/SagaService.kt b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/SagaService.kt index 833387ff6d7..13adeebef42 100644 --- a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/SagaService.kt +++ b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/SagaService.kt @@ -70,7 +70,7 @@ class SagaService( private val actionInvocationsId = registry.createId("sagas.actions.invocations") fun applyBlocking(sagaName: String, sagaId: String, flow: SagaFlow, startingCommand: SagaCommand): T? { - val initialSaga = initializeSaga(startingCommand) + val initialSaga = initializeSaga(startingCommand, sagaName, sagaId) log.info("Applying saga: ${initialSaga.name}/${initialSaga.id}") @@ -119,7 +119,7 @@ class SagaService( "Failed to apply action ${action.javaClass.simpleName} for ${saga.name}/${saga.id}", e) } - saga.setSequence(stepCommand.metadata.sequence) + saga.setSequence(stepCommand.getMetadata().sequence) val newEvents: MutableList = result.events.toMutableList().also { it.add(SagaCommandCompleted(getStepCommandName(stepCommand))) @@ -149,32 +149,32 @@ class SagaService( return invokeCompletionHandler(initialSaga, flow) } - private fun initializeSaga(command: SagaCommand): Saga { - return sagaRepository.get(command.sagaName, command.sagaId) - ?: Saga(command.sagaName, command.sagaId) + private fun initializeSaga(command: SagaCommand, sagaName: String, sagaId: String): Saga { + return sagaRepository.get(sagaName, sagaId) + ?: Saga(sagaName, sagaId) .also { - log.debug("Initializing new saga: ${it.name}/${it.id}") + log.debug("Initializing new saga: $sagaName/$sagaId") it.addEvent(command) sagaRepository.save(it) } } private fun invokeCompletionHandler(saga: Saga, flow: SagaFlow): T? { - if (flow.completionHandler != null) { - val handler = applicationContext.getBean(flow.completionHandler) - val result = sagaRepository.get(saga.name, saga.id) - ?.let { handler.handle(it) } - ?: throw SagaNotFoundException("Could not find Saga to complete by ${saga.name}/${saga.id}") - - // TODO(rz): Haha... :( - try { - @Suppress("UNCHECKED_CAST") - return result as T? - } catch (e: ClassCastException) { - throw SagaIntegrationException("The completion handler is incompatible with the expected return type", e) + return flow.completionHandler + ?.let { completionHandler -> + val handler = applicationContext.getBean(completionHandler) + val result = sagaRepository.get(saga.name, saga.id) + ?.let { handler.handle(it) } + ?: throw SagaNotFoundException("Could not find Saga to complete by ${saga.name}/${saga.id}") + + // TODO(rz): Haha... :( + try { + @Suppress("UNCHECKED_CAST") + return result as T? + } catch (e: ClassCastException) { + throw SagaIntegrationException("The completion handler is incompatible with the expected return type", e) + } } - } - return null } private fun getRequiredCommand(action: SagaAction): Class { diff --git a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/config/SagaAutoConfiguration.kt b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/config/SagaAutoConfiguration.kt index f107f95782a..50564711f7b 100644 --- a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/config/SagaAutoConfiguration.kt +++ b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/config/SagaAutoConfiguration.kt @@ -16,11 +16,14 @@ package com.netflix.spinnaker.clouddriver.saga.config import com.netflix.spectator.api.Registry +import com.netflix.spinnaker.clouddriver.event.SpinnakerEvent import com.netflix.spinnaker.clouddriver.event.config.EventSourceAutoConfiguration import com.netflix.spinnaker.clouddriver.event.persistence.EventRepository import com.netflix.spinnaker.clouddriver.saga.SagaService import com.netflix.spinnaker.clouddriver.saga.persistence.DefaultSagaRepository import com.netflix.spinnaker.clouddriver.saga.persistence.SagaRepository +import com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer.ClassSubtypeLocator +import com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer.SubtypeLocator import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -47,6 +50,14 @@ open class SagaAutoConfiguration { registry: Registry ): SagaService = SagaService(sagaRepository, registry) + + @Bean + open fun sagaEventSubtypeLocator(): SubtypeLocator { + return ClassSubtypeLocator( + SpinnakerEvent::class.java, + listOf("com.netflix.spinnaker.clouddriver.saga") + ) + } } @ConfigurationProperties("spinnaker.clouddriver.sagas") diff --git a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/events.kt b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/events.kt index 12ed03e6a01..2605462649b 100644 --- a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/events.kt +++ b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/events.kt @@ -17,22 +17,22 @@ package com.netflix.spinnaker.clouddriver.saga import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonTypeName -import com.netflix.spinnaker.clouddriver.event.SpinnakerEvent +import com.netflix.spinnaker.clouddriver.event.AbstractSpinnakerEvent import com.netflix.spinnaker.clouddriver.saga.models.Saga import com.netflix.spinnaker.kork.exceptions.SpinnakerException /** * Root event type for [Saga]s. * - * @property sagaName Alias for [aggregateType] - * @property sagaId Alias for [aggregateId] + * @property sagaName Alias for [metadata.aggregateType] + * @property sagaId Alias for [metadata.aggregateId] */ -abstract class SagaEvent : SpinnakerEvent() { +abstract class SagaEvent : AbstractSpinnakerEvent() { val sagaName - @JsonIgnore get() = aggregateType + @JsonIgnore get() = getMetadata().aggregateType val sagaId - @JsonIgnore get() = aggregateId + @JsonIgnore get() = getMetadata().aggregateId } /** @@ -45,7 +45,7 @@ abstract class SagaEvent : SpinnakerEvent() { */ @JsonTypeName("sagaSaved") class SagaSaved( - val saga: Saga + val sequence: Long ) : SagaEvent() /** @@ -160,6 +160,7 @@ class SagaCommandCompleted( * This event is unwrapped prior to being added to the event log; so all [SagaCommand]s defined within this * wrapper will show up as their own distinct log entries. */ +@JsonTypeName("sagaManyCommandsWrapper") class ManyCommands( command1: SagaCommand, vararg extraCommands: SagaCommand diff --git a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/models/Saga.kt b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/models/Saga.kt index 7bdf5b35f29..77c55c86073 100644 --- a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/models/Saga.kt +++ b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/models/Saga.kt @@ -59,7 +59,7 @@ class Saga( fun isCompensating(): Boolean = events.filterIsInstance().isNotEmpty() fun getVersion(): Long { - return events.map { it.metadata.originatingVersion }.max()?.let { it + 1 } ?: 0 + return events.map { it.getMetadata().originatingVersion }.max()?.let { it + 1 } ?: 0 } fun addEvent(event: SagaEvent) { diff --git a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/DefaultSagaRepository.kt b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/DefaultSagaRepository.kt index 0b778074e6c..2f87818fcc2 100644 --- a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/DefaultSagaRepository.kt +++ b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/DefaultSagaRepository.kt @@ -15,7 +15,9 @@ */ package com.netflix.spinnaker.clouddriver.saga.persistence +import com.netflix.spinnaker.clouddriver.event.Aggregate import com.netflix.spinnaker.clouddriver.event.persistence.EventRepository +import com.netflix.spinnaker.clouddriver.event.persistence.EventRepository.ListAggregatesCriteria import com.netflix.spinnaker.clouddriver.saga.SagaEvent import com.netflix.spinnaker.clouddriver.saga.SagaSaved import com.netflix.spinnaker.clouddriver.saga.models.Saga @@ -33,9 +35,17 @@ class DefaultSagaRepository( override fun list(criteria: SagaRepository.ListCriteria): List { val sagas = if (criteria.names != null && criteria.names.isNotEmpty()) { - criteria.names.flatMap { eventRepository.listAggregates(it) } + var token: String? = null + val aggregates: MutableList = mutableListOf() + do { + eventRepository.listAggregates(ListAggregatesCriteria(token = token, perPage = 1_000)).let { + aggregates.addAll(it.aggregates) + token = it.nextPageToken + } + } while (token != null) + aggregates } else { - eventRepository.listAggregates(null) + eventRepository.listAggregates(ListAggregatesCriteria()).aggregates }.mapNotNull { get(it.type, it.id) } return if (criteria.running == null) { @@ -54,14 +64,11 @@ class DefaultSagaRepository( return events .filterIsInstance() .last() - .saga .let { - // Copy the Saga: We don't want to accidentally mutate the saga that's in the event if the - // eventRepository is in-memory only. Saga( - name = it.name, - id = it.id, - sequence = it.getSequence() + name = it.sagaName, + id = it.sagaId, + sequence = it.sequence ) } .also { saga -> @@ -74,7 +81,7 @@ class DefaultSagaRepository( if (additionalEvents.isNotEmpty()) { events.addAll(additionalEvents) } - events.add(SagaSaved(saga)) + events.add(SagaSaved(saga.getSequence())) eventRepository.save(saga.name, saga.id, saga.getVersion(), events) } } diff --git a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/SagaRepository.kt b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/SagaRepository.kt index d33260900b4..fd48d675145 100644 --- a/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/SagaRepository.kt +++ b/clouddriver-saga/src/main/kotlin/com/netflix/spinnaker/clouddriver/saga/persistence/SagaRepository.kt @@ -26,6 +26,8 @@ interface SagaRepository { /** * List all [Saga]s that match the provided [criteria]. * + * TODO(rz): Support pagination + * * @param criteria Query criteria for Sagas * @return A list of matching Sagas */ diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/RegionScopedTitusClient.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/RegionScopedTitusClient.java index 7a2bbb7f8c3..f39899cb396 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/RegionScopedTitusClient.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/RegionScopedTitusClient.java @@ -209,13 +209,13 @@ public String submitJob(SubmitJobRequest submitJobRequest) { jobDescription.setUser(jobDescription.getUser() + "@netflix.com"); } if (jobDescription.getJobGroupSequence() == null - && jobDescription.getType().equals("service")) { + && "service".equals(jobDescription.getType())) { try { int sequence = Names.parseName(jobDescription.getName()).getSequence(); jobDescription.setJobGroupSequence(String.format("v%03d", sequence)); } catch (Exception e) { - log.error("Cannot get job group sequence", e); - // fail silently if we can't get a job group sequence + // fail silently if we can't get a job group sequence: This is normal if no prior jobs + // exist. } } jobDescription.getLabels().put("name", jobDescription.getName()); diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/JobDescription.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/JobDescription.java index 226d9b0585f..03c368e0545 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/JobDescription.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/JobDescription.java @@ -79,16 +79,16 @@ public class JobDescription { applicationName = request.getDockerImageName(); version = request.getDockerImageVersion(); digest = request.getDockerDigest(); - instancesDesired = request.getInstanceDesired(); - instancesMin = request.getInstanceMin(); - instancesMax = request.getInstanceMax(); + instancesDesired = request.getInstancesDesired(); + instancesMin = request.getInstancesMin(); + instancesMax = request.getInstancesMax(); cpu = request.getCpu(); memory = request.getMemory(); sharedMemory = request.getSharedMemory(); disk = request.getDisk(); ports = request.getPorts(); networkMbps = request.getNetworkMbps(); - allocateIpAddress = request.getAllocateIpAddress(); + allocateIpAddress = request.isAllocateIpAddress(); appName = request.getApplication(); jobGroupStack = request.getStack(); jobGroupDetail = request.getDetail(); diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/SubmitJobRequest.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/SubmitJobRequest.java index 1412d12f8fd..a525d96135e 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/SubmitJobRequest.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/SubmitJobRequest.java @@ -16,13 +16,26 @@ package com.netflix.spinnaker.clouddriver.titus.client.model; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.Builder; import lombok.Data; - +import lombok.Getter; +import lombok.Value; +import lombok.experimental.Wither; + +@JsonDeserialize(builder = SubmitJobRequest.SubmitJobRequestBuilder.class) +@Builder(builderClassName = "SubmitJobRequestBuilder", toBuilder = true) +@Wither +@Value public class SubmitJobRequest { + @Getter + @Value public static class Constraint { enum ConstraintType { SOFT, @@ -42,24 +55,10 @@ public static Constraint soft(String constraint) { private final ConstraintType constraintType; private final String constraint; - - public Constraint(ConstraintType constraintType, String constraint) { - this.constraintType = constraintType; - this.constraint = constraint; - } - - public ConstraintType getConstraintType() { - return constraintType; - } - - public String getConstraint() { - return constraint; - } } @Data public static class Constraints { - public Map hard; public Map soft; } @@ -77,7 +76,7 @@ public static class Constraints { private String entryPoint; private String iamProfile; private String capacityGroup; - private Boolean inService = true; + @Builder.Default private Boolean inService = true; private int instancesMin; private int instancesMax; private int instancesDesired; @@ -93,357 +92,21 @@ public static class Constraints { private int[] ports; private Map env; private boolean allocateIpAddress; - private List constraints = new ArrayList<>(); - private Map labels = new HashMap(); - private Map containerAttributes = new HashMap(); - private List securityGroups = null; - private MigrationPolicy migrationPolicy = null; - private DisruptionBudget disruptionBudget = null; - - private Constraints containerConstraints = null; - private ServiceJobProcesses serviceJobProcesses = null; - - public DisruptionBudget getDisruptionBudget() { - return disruptionBudget; - } - - public SubmitJobRequest withJobType(String jobType) { - this.jobType = jobType; - return this; - } - - public SubmitJobRequest withJobName(String jobName) { - this.jobName = jobName; - return this; - } - - public SubmitJobRequest withApplication(String application) { - this.application = application; - return this; - } - - public SubmitJobRequest withDockerImageName(String dockerImageName) { - this.dockerImageName = dockerImageName; - return this; - } - - public SubmitJobRequest withDockerImageVersion(String dockerImageVersion) { - this.dockerImageVersion = dockerImageVersion; - return this; - } - - public SubmitJobRequest withDockerDigest(String dockerDigest) { - this.dockerDigest = dockerDigest; - return this; - } - - public SubmitJobRequest withInstancesMin(int instancesMin) { - this.instancesMin = instancesMin; - return this; - } - - public SubmitJobRequest withInstancesMax(int instancesMax) { - this.instancesMax = instancesMax; - return this; - } - - public SubmitJobRequest withInstancesDesired(int instancesDesired) { - this.instancesDesired = instancesDesired; - return this; - } - - public SubmitJobRequest withCpu(int cpu) { - this.cpu = cpu; - return this; - } - - public SubmitJobRequest withMemory(int memory) { - this.memory = memory; - return this; - } - - public SubmitJobRequest withSharedMemory(int sharedMemory) { - this.sharedMemory = sharedMemory; - return this; - } - - public SubmitJobRequest withDisk(int disk) { - this.disk = disk; - return this; - } - - public SubmitJobRequest withRetries(int retries) { - this.retries = retries; - return this; - } - - public SubmitJobRequest withRuntimeLimitSecs(int runtimeLimitSecs) { - this.runtimeLimitSecs = runtimeLimitSecs; - return this; - } - - public SubmitJobRequest withGpu(int gpu) { - this.gpu = gpu; - return this; - } - - public SubmitJobRequest withPorts(int[] ports) { - this.ports = ports; - return this; - } - - public SubmitJobRequest withNetworkMbps(int networkMbps) { - this.networkMbps = networkMbps; - return this; - } - - public SubmitJobRequest withEnv(Map env) { - this.env = env; - return this; - } - - public SubmitJobRequest withAllocateIpAddress(boolean allocateIpAddress) { - this.allocateIpAddress = allocateIpAddress; - return this; - } - - public SubmitJobRequest withStack(String stack) { - this.stack = stack; - return this; - } - - public SubmitJobRequest withDetail(String detail) { - this.detail = detail; - return this; - } - - public SubmitJobRequest withUser(String user) { - this.user = user; - return this; - } - - public SubmitJobRequest withEntryPoint(String entryPoint) { - this.entryPoint = entryPoint; - return this; - } - - public SubmitJobRequest withIamProfile(String iamProfile) { - this.iamProfile = iamProfile; - return this; - } - - public SubmitJobRequest withSecurityGroups(List securityGroups) { - this.securityGroups = securityGroups; - return this; - } - - public SubmitJobRequest withCapacityGroup(String capacityGroup) { - this.capacityGroup = capacityGroup; - return this; - } - - public SubmitJobRequest withConstraint(Constraint constraint) { - this.constraints.add(constraint); - return this; - } - - public SubmitJobRequest withLabels(Map labels) { - this.labels = labels; - return this; - } - - public SubmitJobRequest withContainerAttributes(Map containerAttributes) { - this.containerAttributes = containerAttributes; - return this; - } - - public SubmitJobRequest withLabel(String key, String value) { - this.labels.put(key, value); - return this; - } - - public SubmitJobRequest withInService(Boolean inService) { - this.inService = inService; - return this; - } - - public SubmitJobRequest withMigrationPolicy(MigrationPolicy migrationPolicy) { - this.migrationPolicy = migrationPolicy; - return this; - } - - public SubmitJobRequest withEfs(Efs efs) { - this.efs = efs; - return this; - } + @Builder.Default private List constraints = new ArrayList<>(); + @Builder.Default private Map labels = new HashMap(); + @Builder.Default private Map containerAttributes = new HashMap(); + @Builder.Default private List securityGroups = null; + @Builder.Default private MigrationPolicy migrationPolicy = null; + @Builder.Default private DisruptionBudget disruptionBudget = null; - public SubmitJobRequest withCredentials(String credentials) { - this.credentials = credentials; - return this; - } - - public SubmitJobRequest withDisruptionBudget(DisruptionBudget disruptionBudget) { - this.disruptionBudget = disruptionBudget; - return this; - } - - public SubmitJobRequest withConstraints(Constraints constraints) { - this.containerConstraints = constraints; - return this; - } - - public SubmitJobRequest withServiceJobProcesses(ServiceJobProcesses serviceJobProcesses) { - this.serviceJobProcesses = serviceJobProcesses; - return this; - } - - // Getters - - public String getJobType() { - return jobType; - } - - public int getInstanceMin() { - return instancesMin; - } - - public int getInstanceMax() { - return instancesMax; - } - - public int getInstanceDesired() { - return instancesDesired; - } - - public int getCpu() { - return cpu; - } - - public int getGpu() { - return gpu; - } - - public int getRetries() { - return retries; - } - - public int getRuntimeLimitSecs() { - return runtimeLimitSecs; - } - - public int getMemory() { - return memory; - } - - public int getSharedMemory() { - return sharedMemory; - } - - public int getDisk() { - return disk; - } - - public int getNetworkMbps() { - return networkMbps; - } - - public int[] getPorts() { - return ports; - } - - public Map getEnv() { - return env; - } - - public String getApplication() { - return application; - } - - public String getJobName() { - return jobName; - } - - public String getDockerImageName() { - return dockerImageName; - } - - public String getDockerImageVersion() { - return dockerImageVersion; - } - - public String getDockerDigest() { - return dockerDigest; - } - - public boolean getAllocateIpAddress() { - return allocateIpAddress; - } - - public String getStack() { - return stack; - } - - public String getDetail() { - return detail; - } - - public String getUser() { - return user; - } - - public List getConstraints() { - return constraints; - } - - public List getSecurityGroups() { - return securityGroups; - } - - public String getEntryPoint() { - return entryPoint; - } - - public String getIamProfile() { - return iamProfile; - } - - public Boolean getInService() { - return inService; - } - - public String getCapacityGroup() { - return capacityGroup; - } - - public Map getLabels() { - return labels; - } - - public Map getContainerAttributes() { - return containerAttributes; - } + @Builder.Default private Constraints containerConstraints = null; + @Builder.Default private ServiceJobProcesses serviceJobProcesses = null; + @JsonIgnore public JobDescription getJobDescription() { return new JobDescription(this); } - public Efs getEfs() { - return efs; - } - - public String getCredentials() { - return credentials; - } - - public MigrationPolicy getMigrationPolicy() { - return migrationPolicy; - } - - public Constraints getContainerConstraints() { - return containerConstraints; - } - - public ServiceJobProcesses getServiceJobProcesses() { - return serviceJobProcesses; - } + @JsonPOJOBuilder(withPrefix = "") + public static class SubmitJobRequestBuilder {} } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AbstractTitusDeployAction.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AbstractTitusDeployAction.java index 70ab15c5c62..0f3e42bdd7e 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AbstractTitusDeployAction.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AbstractTitusDeployAction.java @@ -71,4 +71,14 @@ TitusClient buildSourceTitusClient(TitusDeployDescription.Source source) { } return null; } + + /** Re-sets credentials into a deserialized {@code TitusDeployDescription}. */ + void prepareDeployDescription(final TitusDeployDescription description) { + if (description.getCredentials() == null) { + AccountCredentials credentials = + accountCredentialsRepository.getOne(description.getAccount()); + TitusUtils.assertTitusAccountCredentialsType(credentials); + description.setCredentials((NetflixTitusCredentials) credentials); + } + } } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AttachTitusServiceLoadBalancers.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AttachTitusServiceLoadBalancers.java index f6de12f4d18..e8358df712e 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AttachTitusServiceLoadBalancers.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/AttachTitusServiceLoadBalancers.java @@ -15,31 +15,40 @@ */ package com.netflix.spinnaker.clouddriver.titus.deploy.actions; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.netflix.spinnaker.clouddriver.aws.deploy.ops.loadbalancer.TargetGroupLookupHelper; import com.netflix.spinnaker.clouddriver.saga.SagaCommand; import com.netflix.spinnaker.clouddriver.saga.flow.SagaAction; import com.netflix.spinnaker.clouddriver.saga.models.Saga; +import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository; import com.netflix.spinnaker.clouddriver.titus.TitusClientProvider; import com.netflix.spinnaker.clouddriver.titus.client.TitusLoadBalancerClient; import com.netflix.spinnaker.clouddriver.titus.deploy.description.TitusDeployDescription; import com.netflix.spinnaker.clouddriver.titus.deploy.events.TitusLoadBalancerAttached; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.Builder; import lombok.EqualsAndHashCode; -import lombok.Getter; +import lombok.Value; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class AttachTitusServiceLoadBalancers +public class AttachTitusServiceLoadBalancers extends AbstractTitusDeployAction implements SagaAction { private final TitusClientProvider titusClientProvider; @Autowired - public AttachTitusServiceLoadBalancers(TitusClientProvider titusClientProvider) { - this.titusClientProvider = titusClientProvider; + public AttachTitusServiceLoadBalancers( + AccountCredentialsRepository accountCredentialsRepository, + TitusClientProvider titusClientProvider, + TitusClientProvider titusClientProvider1) { + super(accountCredentialsRepository, titusClientProvider); + this.titusClientProvider = titusClientProvider1; } @NotNull @@ -47,6 +56,8 @@ public AttachTitusServiceLoadBalancers(TitusClientProvider titusClientProvider) public Result apply(@NotNull AttachTitusServiceLoadBalancersCommand command, @NotNull Saga saga) { final TitusDeployDescription description = command.description; + prepareDeployDescription(description); + TitusLoadBalancerClient loadBalancerClient = titusClientProvider.getTitusLoadBalancerClient( description.getCredentials(), description.getRegion()); @@ -68,7 +79,11 @@ public Result apply(@NotNull AttachTitusServiceLoadBalancersCommand command, @No targetGroupArn -> { loadBalancerClient.addLoadBalancer(jobUri, targetGroupArn); saga.log("Attached %s to %s", targetGroupArn, jobUri); - saga.addEvent(new TitusLoadBalancerAttached(jobUri, targetGroupArn)); + saga.addEvent( + TitusLoadBalancerAttached.builder() + .jobUri(jobUri) + .targetGroupArn(targetGroupArn) + .build()); }); saga.log("Load balancers applied"); @@ -77,21 +92,20 @@ public Result apply(@NotNull AttachTitusServiceLoadBalancersCommand command, @No return new Result(); } + @Builder(builderClassName = "AttachTitusServiceLoadBalancersCommandBuilder", toBuilder = true) + @JsonDeserialize( + builder = + AttachTitusServiceLoadBalancersCommand.AttachTitusServiceLoadBalancersCommandBuilder + .class) + @JsonTypeName("attachTitusServiceLoadBalancersCommand") @EqualsAndHashCode(callSuper = true) - @Getter + @Value public static class AttachTitusServiceLoadBalancersCommand extends SagaCommand { - @Nonnull private final TitusDeployDescription description; - @Nonnull private final String jobUri; - @Nullable private final TargetGroupLookupHelper.TargetGroupLookupResult targetGroupLookupResult; + @Nonnull private TitusDeployDescription description; + @Nonnull private String jobUri; + @Nullable private TargetGroupLookupHelper.TargetGroupLookupResult targetGroupLookupResult; - public AttachTitusServiceLoadBalancersCommand( - @Nonnull TitusDeployDescription description, - @Nonnull String jobUri, - @Nullable TargetGroupLookupHelper.TargetGroupLookupResult targetGroupLookupResult) { - super(); - this.description = description; - this.jobUri = jobUri; - this.targetGroupLookupResult = targetGroupLookupResult; - } + @JsonPOJOBuilder(withPrefix = "") + public static class AttachTitusServiceLoadBalancersCommandBuilder {} } } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/CopyTitusServiceScalingPolicies.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/CopyTitusServiceScalingPolicies.java index a6a2314fc70..5aeb89baf83 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/CopyTitusServiceScalingPolicies.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/CopyTitusServiceScalingPolicies.java @@ -15,6 +15,9 @@ */ package com.netflix.spinnaker.clouddriver.titus.deploy.actions; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.netflix.spinnaker.clouddriver.saga.SagaCommand; import com.netflix.spinnaker.clouddriver.saga.flow.SagaAction; import com.netflix.spinnaker.clouddriver.saga.models.Saga; @@ -38,8 +41,9 @@ import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; +import lombok.Builder; import lombok.EqualsAndHashCode; -import lombok.Getter; +import lombok.Value; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -63,6 +67,8 @@ public CopyTitusServiceScalingPolicies( public Result apply(@NotNull CopyTitusServiceScalingPoliciesCommand command, @NotNull Saga saga) { final TitusDeployDescription description = command.description; + prepareDeployDescription(description); + if (!description.isCopySourceScalingPolicies() || !description.getCopySourceScalingPoliciesAndActions()) { saga.log("Not applying scaling policies: None to apply"); @@ -117,8 +123,11 @@ public Result apply(@NotNull CopyTitusServiceScalingPoliciesCommand command, @No .toScalingPolicyBuilder()); autoscalingClient.createScalingPolicy(builder.build()); saga.addEvent( - new TitusScalingPolicyCopied( - serverGroupName, description.getRegion(), policy.getId().getId())); + TitusScalingPolicyCopied.builder() + .serverGroupName(serverGroupName) + .region(description.getRegion()) + .sourcePolicyId(policy.getId().getId()) + .build()); } }); } @@ -149,21 +158,20 @@ private TitusAutoscalingClient buildSourceAutoscalingClient( source.getAsgName()); } + @Builder(builderClassName = "CopyTitusServiceScalingPoliciesCommandBuilder", toBuilder = true) + @JsonDeserialize( + builder = + CopyTitusServiceScalingPoliciesCommand.CopyTitusServiceScalingPoliciesCommandBuilder + .class) + @JsonTypeName("copyTitusServiceScalingPoliciesCommand") @EqualsAndHashCode(callSuper = true) - @Getter + @Value public static class CopyTitusServiceScalingPoliciesCommand extends SagaCommand { - @Nonnull private final TitusDeployDescription description; - @Nonnull private final String jobUri; - @Nonnull private final String deployedServerGroupName; - - public CopyTitusServiceScalingPoliciesCommand( - @Nonnull TitusDeployDescription description, - @Nonnull String jobUri, - @Nonnull String deployedServerGroupName) { - super(); - this.description = description; - this.jobUri = jobUri; - this.deployedServerGroupName = deployedServerGroupName; - } + @Nonnull private TitusDeployDescription description; + @Nonnull private String jobUri; + @Nonnull private String deployedServerGroupName; + + @JsonPOJOBuilder(withPrefix = "") + public static class CopyTitusServiceScalingPoliciesCommandBuilder {} } } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/LoadFront50App.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/LoadFront50App.java index fa166b982bd..49347ad2766 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/LoadFront50App.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/LoadFront50App.java @@ -17,8 +17,13 @@ import static java.lang.String.format; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.netflix.spinnaker.clouddriver.core.services.Front50Service; +import com.netflix.spinnaker.clouddriver.event.CompositeSpinnakerEvent; +import com.netflix.spinnaker.clouddriver.event.SpinnakerEvent; import com.netflix.spinnaker.clouddriver.saga.ManyCommands; import com.netflix.spinnaker.clouddriver.saga.SagaCommand; import com.netflix.spinnaker.clouddriver.saga.exceptions.SagaIntegrationException; @@ -26,13 +31,14 @@ import com.netflix.spinnaker.clouddriver.saga.models.Saga; import com.netflix.spinnaker.kork.exceptions.SystemException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; import lombok.AccessLevel; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.EqualsAndHashCode; -import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Value; import org.jetbrains.annotations.NotNull; @@ -127,25 +133,29 @@ public Result apply(@NotNull LoadFront50AppCommand command, @NotNull Saga saga) } } + /** Marks a SagaCommand as being aware of the result of the LoadFront50App SagaAction. */ + interface Front50AppAware { + void setFront50App(Front50App app); + } + + @Builder(builderClassName = "LoadFront50AppCommandBuilder", toBuilder = true) + @JsonDeserialize(builder = LoadFront50AppCommand.LoadFront50AppCommandBuilder.class) + @JsonTypeName("loadFront50AppCommand") @EqualsAndHashCode(callSuper = true) - @Getter - public static class LoadFront50AppCommand extends SagaCommand { - @Nonnull private final String appName; - @Nonnull private final SagaCommand nextCommand; - @Nonnull private final boolean allowMissing; + @Value + public static class LoadFront50AppCommand extends SagaCommand implements CompositeSpinnakerEvent { + @Nonnull private String appName; + @Nonnull private SagaCommand nextCommand; + private boolean allowMissing; - public LoadFront50AppCommand( - @Nonnull String appName, @Nonnull SagaCommand nextCommand, boolean allowMissing) { - super(); - this.appName = appName; - this.nextCommand = nextCommand; - this.allowMissing = allowMissing; + @NotNull + @Override + public List getComposedEvents() { + return Collections.singletonList(nextCommand); } - } - /** Marks a SagaCommand as being aware of the result of the LoadFront50App SagaAction. */ - interface Front50AppAware { - void setFront50App(Front50App app); + @JsonPOJOBuilder(withPrefix = "") + public static class LoadFront50AppCommandBuilder {} } @Value diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/PrepareTitusDeploy.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/PrepareTitusDeploy.java index f1458b2316f..2cca39db648 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/PrepareTitusDeploy.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/PrepareTitusDeploy.java @@ -17,6 +17,9 @@ import static java.lang.String.format; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.netflix.frigga.Names; @@ -36,7 +39,6 @@ import com.netflix.spinnaker.clouddriver.titus.client.TitusClient; import com.netflix.spinnaker.clouddriver.titus.client.model.DisruptionBudget; import com.netflix.spinnaker.clouddriver.titus.client.model.Job; -import com.netflix.spinnaker.clouddriver.titus.client.model.SubmitJobRequest; import com.netflix.spinnaker.clouddriver.titus.client.model.disruption.AvailabilityPercentageLimit; import com.netflix.spinnaker.clouddriver.titus.client.model.disruption.ContainerHealthProvider; import com.netflix.spinnaker.clouddriver.titus.client.model.disruption.HourlyTimeWindow; @@ -56,6 +58,7 @@ import java.util.Set; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Value; import lombok.experimental.NonFinal; @@ -118,6 +121,8 @@ private static int orDefault(int input, int defaultValue) { public Result apply(@NotNull PrepareTitusDeployCommand command, @NotNull Saga saga) { final TitusDeployDescription description = command.description; + prepareDeployDescription(description); + final TitusClient titusClient = titusClientProvider.getTitusClient( description.getCredentials(), command.description.getRegion()); @@ -150,14 +155,6 @@ public Result apply(@NotNull PrepareTitusDeployCommand command, @NotNull Saga sa resolveSecurityGroups(saga, description); - SubmitJobRequest submitJobRequest = description.toSubmitJobRequest(dockerImage); - - if (front50App != null && !isNullOrEmpty(front50App.getEmail())) { - submitJobRequest.withUser(front50App.getEmail()); - } else if (!isNullOrEmpty(description.getUser())) { - submitJobRequest.withUser(description.getUser()); - } - TargetGroupLookupHelper.TargetGroupLookupResult targetGroupLookupResult = null; if (!description.getTargetGroups().isEmpty()) { targetGroupLookupResult = validateLoadBalancers(description); @@ -172,16 +169,33 @@ public Result apply(@NotNull PrepareTitusDeployCommand command, @NotNull Saga sa description.getLabels().remove(LABEL_TARGET_GROUPS); } - String nextServerGroupName = - TitusJobNameResolver.resolveJobName(titusClient, description, submitJobRequest); + String nextServerGroupName = TitusJobNameResolver.resolveJobName(titusClient, description); saga.log("Resolved server group name to %s", nextServerGroupName); + String user = resolveUser(front50App, description); + return new Result( - new SubmitTitusJobCommand( - description, submitJobRequest, nextServerGroupName, targetGroupLookupResult), + SubmitTitusJobCommand.builder() + .description(description) + .submitJobRequest( + description.toSubmitJobRequest(dockerImage, nextServerGroupName, user)) + .nextServerGroupName(nextServerGroupName) + .targetGroupLookupResult(targetGroupLookupResult) + .build(), Collections.emptyList()); } + @Nullable + private String resolveUser( + LoadFront50App.Front50App front50App, TitusDeployDescription description) { + if (front50App != null && !isNullOrEmpty(front50App.getEmail())) { + return front50App.getEmail(); + } else if (!isNullOrEmpty(description.getUser())) { + return description.getUser(); + } + return null; + } + private void configureDisruptionBudget( TitusDeployDescription description, Job sourceJob, LoadFront50App.Front50App front50App) { if (description.getDisruptionBudget() == null) { @@ -438,21 +452,22 @@ private void resolveSecurityGroups(Saga saga, TitusDeployDescription description Joiner.on(",").join(description.getSecurityGroups())); } - @EqualsAndHashCode(callSuper = true) + @Builder(builderClassName = "PrepareTitusDeployCommandBuilder", toBuilder = true) + @JsonDeserialize(builder = PrepareTitusDeployCommand.PrepareTitusDeployCommandBuilder.class) + @JsonTypeName("prepareTitusDeployCommand") @Value + @EqualsAndHashCode(callSuper = true) public static class PrepareTitusDeployCommand extends SagaCommand implements Front50AppAware { - private final TitusDeployDescription description; + private TitusDeployDescription description; @NonFinal private LoadFront50App.Front50App front50App; - public PrepareTitusDeployCommand(TitusDeployDescription description) { - super(); - this.description = description; - } - @Override public void setFront50App(LoadFront50App.Front50App app) { this.front50App = app; } + + @JsonPOJOBuilder(withPrefix = "") + public static class PrepareTitusDeployCommandBuilder {} } private static class SecurityGroupNotFoundException extends TitusException { diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/SubmitTitusJob.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/SubmitTitusJob.java index 3c2ea4ec521..9381faddd64 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/SubmitTitusJob.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/SubmitTitusJob.java @@ -18,11 +18,15 @@ import static com.netflix.spinnaker.clouddriver.titus.deploy.actions.AttachTitusServiceLoadBalancers.AttachTitusServiceLoadBalancersCommand; import static com.netflix.spinnaker.clouddriver.titus.deploy.actions.CopyTitusServiceScalingPolicies.CopyTitusServiceScalingPoliciesCommand; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.netflix.spinnaker.clouddriver.aws.deploy.ops.loadbalancer.TargetGroupLookupHelper; import com.netflix.spinnaker.clouddriver.saga.ManyCommands; import com.netflix.spinnaker.clouddriver.saga.SagaCommand; import com.netflix.spinnaker.clouddriver.saga.flow.SagaAction; import com.netflix.spinnaker.clouddriver.saga.models.Saga; +import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository; import com.netflix.spinnaker.clouddriver.titus.JobType; import com.netflix.spinnaker.clouddriver.titus.TitusClientProvider; import com.netflix.spinnaker.clouddriver.titus.TitusException; @@ -36,9 +40,9 @@ import io.grpc.StatusRuntimeException; import java.util.Collections; import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import lombok.Builder; import lombok.EqualsAndHashCode; -import lombok.Getter; +import lombok.Value; import lombok.experimental.NonFinal; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -47,36 +51,22 @@ import org.springframework.stereotype.Component; @Component -public class SubmitTitusJob implements SagaAction { +public class SubmitTitusJob extends AbstractTitusDeployAction + implements SagaAction { private static final Logger log = LoggerFactory.getLogger(SubmitTitusJob.class); - private final TitusClientProvider titusClientProvider; private final RetrySupport retrySupport; @Autowired - public SubmitTitusJob(TitusClientProvider titusClientProvider, RetrySupport retrySupport) { - this.titusClientProvider = titusClientProvider; + public SubmitTitusJob( + AccountCredentialsRepository accountCredentialsRepository, + TitusClientProvider titusClientProvider, + RetrySupport retrySupport) { + super(accountCredentialsRepository, titusClientProvider); this.retrySupport = retrySupport; } - private static boolean isServiceExceptionRetryable( - TitusDeployDescription description, StatusRuntimeException e) { - String statusDescription = e.getStatus().getDescription(); - return JobType.isEqual(description.getJobType(), JobType.SERVICE) - && (e.getStatus().getCode() == Status.RESOURCE_EXHAUSTED.getCode() - || e.getStatus().getCode() == Status.INVALID_ARGUMENT.getCode()) - && (statusDescription != null - && (statusDescription.contains("Job sequence id reserved by another pending job") - || statusDescription.contains("Constraint violation - job with group sequence"))); - } - - private static boolean isStatusCodeRetryable(Status.Code code) { - return code == Status.UNAVAILABLE.getCode() - || code == Status.INTERNAL.getCode() - || code == Status.DEADLINE_EXCEEDED.getCode(); - } - /** * NOTE: The single-element array usage is to get around line-for-line Groovy conversion variable * references inside of the lambda. This should really be refactored so that pattern isn't @@ -87,6 +77,8 @@ private static boolean isStatusCodeRetryable(Status.Code code) { public Result apply(@NotNull SubmitTitusJobCommand command, @NotNull Saga saga) { final TitusDeployDescription description = command.description; + prepareDeployDescription(description); + final TitusClient titusClient = titusClientProvider.getTitusClient(description.getCredentials(), description.getRegion()); @@ -98,7 +90,7 @@ public Result apply(@NotNull SubmitTitusJobCommand command, @NotNull Saga saga) retrySupport.retry( () -> { try { - return titusClient.submitJob(submitJobRequest); + return titusClient.submitJob(submitJobRequest.withJobName(nextServerGroupName[0])); } catch (StatusRuntimeException e) { if (isServiceExceptionRetryable(description, e)) { String statusDescription = e.getStatus().getDescription(); @@ -114,8 +106,7 @@ public Result apply(@NotNull SubmitTitusJobCommand command, @NotNull Saga saga) retryCount[0]++; } nextServerGroupName[0] = - TitusJobNameResolver.resolveJobName( - titusClient, description, submitJobRequest); + TitusJobNameResolver.resolveJobName(titusClient, description); saga.log("Resolved server group name to '%s'", nextServerGroupName[0]); @@ -147,41 +138,66 @@ public Result apply(@NotNull SubmitTitusJobCommand command, @NotNull Saga saga) return new Result( new ManyCommands( - new AttachTitusServiceLoadBalancersCommand( - description, jobUri, command.targetGroupLookupResult), - new CopyTitusServiceScalingPoliciesCommand( - description, jobUri, nextServerGroupName[0])), + AttachTitusServiceLoadBalancersCommand.builder() + .description(description) + .jobUri(jobUri) + .targetGroupLookupResult(command.targetGroupLookupResult) + .build(), + CopyTitusServiceScalingPoliciesCommand.builder() + .description(description) + .jobUri(jobUri) + .deployedServerGroupName(nextServerGroupName[0]) + .build()), Collections.singletonList( - new TitusJobSubmitted( - Collections.singletonMap(description.getRegion(), nextServerGroupName[0]), - jobUri, - JobType.from(description.getJobType())))); + TitusJobSubmitted.builder() + .jobType(JobType.from(description.getJobType())) + .serverGroupNameByRegion( + Collections.singletonMap(description.getRegion(), nextServerGroupName[0])) + .jobUri(jobUri) + .build())); } + /** + * TODO(rz): Figure out what conditions are not retryable and why. Then document, because what? + */ + private static boolean isServiceExceptionRetryable( + TitusDeployDescription description, StatusRuntimeException e) { + String statusDescription = e.getStatus().getDescription(); + return JobType.SERVICE.isEqual(description.getJobType()) + && (e.getStatus().getCode() == Status.RESOURCE_EXHAUSTED.getCode() + || e.getStatus().getCode() == Status.INVALID_ARGUMENT.getCode()) + && (statusDescription != null + && (statusDescription.contains("Job sequence id reserved by another pending job") + || statusDescription.contains("Constraint violation - job with group sequence"))); + } + + /** + * TODO(rz): Figure out what conditions are not retryable and why. Then document, because what? + */ + private static boolean isStatusCodeRetryable(Status.Code code) { + return code == Status.UNAVAILABLE.getCode() + || code == Status.INTERNAL.getCode() + || code == Status.DEADLINE_EXCEEDED.getCode(); + } + + @Builder(builderClassName = "SubmitTitusJobCommandBuilder", toBuilder = true) + @JsonDeserialize(builder = SubmitTitusJobCommand.SubmitTitusJobCommandBuilder.class) + @JsonTypeName("submitTitusJobCommand") @EqualsAndHashCode(callSuper = true) - @Getter + @Value public static class SubmitTitusJobCommand extends SagaCommand implements Front50AppAware { - @Nonnull private final TitusDeployDescription description; - @Nonnull private final SubmitJobRequest submitJobRequest; - @Nonnull private final String nextServerGroupName; - @Nullable private final TargetGroupLookupHelper.TargetGroupLookupResult targetGroupLookupResult; - @Nullable @NonFinal private LoadFront50App.Front50App front50App; - - public SubmitTitusJobCommand( - @Nonnull TitusDeployDescription description, - @Nonnull SubmitJobRequest submitJobRequest, - @Nonnull String nextServerGroupName, - @Nullable TargetGroupLookupHelper.TargetGroupLookupResult targetGroupLookupResult) { - super(); - this.description = description; - this.submitJobRequest = submitJobRequest; - this.nextServerGroupName = nextServerGroupName; - this.targetGroupLookupResult = targetGroupLookupResult; - } + @Nonnull private TitusDeployDescription description; + @Nonnull private SubmitJobRequest submitJobRequest; + @Nonnull private String nextServerGroupName; + private TargetGroupLookupHelper.TargetGroupLookupResult targetGroupLookupResult; + @NonFinal private LoadFront50App.Front50App front50App; @Override public void setFront50App(LoadFront50App.Front50App app) { this.front50App = app; } + + @JsonPOJOBuilder(withPrefix = "") + public static class SubmitTitusJobCommandBuilder {} } } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusJobNameResolver.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusJobNameResolver.java index 49c2f106565..ac05fc74093 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusJobNameResolver.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusJobNameResolver.java @@ -17,19 +17,14 @@ import com.netflix.spinnaker.clouddriver.titus.JobType; import com.netflix.spinnaker.clouddriver.titus.client.TitusClient; -import com.netflix.spinnaker.clouddriver.titus.client.model.SubmitJobRequest; import com.netflix.spinnaker.clouddriver.titus.deploy.TitusServerGroupNameResolver; import com.netflix.spinnaker.clouddriver.titus.deploy.description.TitusDeployDescription; /** Helper class for resolving Titus job names. */ class TitusJobNameResolver { - static String resolveJobName( - TitusClient titusClient, - TitusDeployDescription description, - SubmitJobRequest submitJobRequest) { - if (JobType.isEqual(submitJobRequest.getJobType(), JobType.BATCH)) { - submitJobRequest.withJobName(description.getApplication()); + static String resolveJobName(TitusClient titusClient, TitusDeployDescription description) { + if (JobType.isEqual(description.getJobType(), JobType.BATCH)) { return description.getApplication(); } @@ -52,7 +47,6 @@ static String resolveJobName( description.getFreeFormDetails(), false); } - submitJobRequest.withJobName(nextServerGroupName); return nextServerGroupName; } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusServiceJobPredicate.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusServiceJobPredicate.java index aa7077e387a..7fd3a5148cc 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusServiceJobPredicate.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/actions/TitusServiceJobPredicate.java @@ -30,10 +30,9 @@ public boolean test(Saga saga) { .filter(e -> PrepareTitusDeployCommand.class.isAssignableFrom(e.getClass())) .findFirst() .map( - e -> { - String jobType = ((PrepareTitusDeployCommand) e).getDescription().getJobType(); - return JobType.isEqual(jobType, JobType.SERVICE); - }) + e -> + JobType.SERVICE.isEqual( + ((PrepareTitusDeployCommand) e).getDescription().getJobType())) .orElseThrow( () -> new TitusException( diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/AbstractTitusCredentialsDescription.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/AbstractTitusCredentialsDescription.java index fd068be4f35..be507cacbd1 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/AbstractTitusCredentialsDescription.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/AbstractTitusCredentialsDescription.java @@ -1,18 +1,20 @@ package com.netflix.spinnaker.clouddriver.titus.deploy.description; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.netflix.spinnaker.clouddriver.security.resources.CredentialsNameable; import com.netflix.spinnaker.clouddriver.titus.credentials.NetflixTitusCredentials; +import java.util.Optional; +@JsonIgnoreProperties("credentials") public abstract class AbstractTitusCredentialsDescription implements CredentialsNameable { - @JsonIgnore private NetflixTitusCredentials credentials; - @JsonProperty("credentials") - public String getCredentialAccount() { - return this.credentials.getName(); - } + private String account; + + private NetflixTitusCredentials credentials; + @JsonIgnore public NetflixTitusCredentials getCredentials() { return credentials; } @@ -20,4 +22,19 @@ public NetflixTitusCredentials getCredentials() { public void setCredentials(NetflixTitusCredentials credentials) { this.credentials = credentials; } + + /** For JSON serde only. */ + @JsonProperty + public void setAccount(String account) { + this.account = account; + } + + /** For JSON serde only. */ + @JsonProperty + @Override + public String getAccount() { + return Optional.ofNullable(this.credentials) + .map(NetflixTitusCredentials::getName) + .orElse(account); + } } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/TitusDeployDescription.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/TitusDeployDescription.java index 0150e2e7e1d..87ea4049d36 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/TitusDeployDescription.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/description/TitusDeployDescription.java @@ -79,48 +79,58 @@ public Collection getApplications() { return Arrays.asList(application); } + /** For Jackson deserialization. */ + public void setApplications(List applications) { + if (!applications.isEmpty()) { + application = applications.get(0); + } + } + @Nonnull - public SubmitJobRequest toSubmitJobRequest(@Nonnull DockerImage dockerImage) { - final SubmitJobRequest submitJobRequest = - new SubmitJobRequest() - .withApplication(application) - .withDockerImageName(dockerImage.getImageName()) - .withInstancesMin(capacity.getMin()) - .withInstancesMax(capacity.getMax()) - .withInstancesDesired(capacity.getDesired()) - .withCpu(resources.getCpu()) - .withMemory(resources.getMemory()) - .withSharedMemory(resources.getSharedMemory()) - .withDisk(resources.getDisk()) - .withRetries(retries) - .withRuntimeLimitSecs(runtimeLimitSecs) - .withGpu(resources.getGpu()) - .withNetworkMbps(resources.getNetworkMbps()) - .withEfs(efs) - .withPorts(resources.getPorts()) - .withEnv(env) - .withAllocateIpAddress(resources.isAllocateIpAddress()) - .withStack(stack) - .withDetail(freeFormDetails) - .withEntryPoint(entryPoint) - .withIamProfile(iamProfile) - .withCapacityGroup(capacityGroup) - .withLabels(labels) - .withInService(inService) - .withMigrationPolicy(migrationPolicy) - .withCredentials(getCredentials().getName()) - .withContainerAttributes(containerAttributes) - .withDisruptionBudget(disruptionBudget) - .withServiceJobProcesses(serviceJobProcesses); + public SubmitJobRequest toSubmitJobRequest( + @Nonnull DockerImage dockerImage, @Nonnull String jobName, String user) { + final SubmitJobRequest.SubmitJobRequestBuilder submitJobRequest = + SubmitJobRequest.builder() + .jobName(jobName) + .user(user) + .application(application) + .dockerImageName(dockerImage.getImageName()) + .instancesMin(capacity.getMin()) + .instancesMax(capacity.getMax()) + .instancesDesired(capacity.getDesired()) + .cpu(resources.getCpu()) + .memory(resources.getMemory()) + .sharedMemory(resources.getSharedMemory()) + .disk(resources.getDisk()) + .retries(retries) + .runtimeLimitSecs(runtimeLimitSecs) + .gpu(resources.getGpu()) + .networkMbps(resources.getNetworkMbps()) + .efs(efs) + .ports(resources.getPorts()) + .env(env) + .allocateIpAddress(resources.isAllocateIpAddress()) + .stack(stack) + .detail(freeFormDetails) + .entryPoint(entryPoint) + .iamProfile(iamProfile) + .capacityGroup(capacityGroup) + .labels(labels) + .inService(inService) + .migrationPolicy(migrationPolicy) + .credentials(getCredentials().getName()) + .containerAttributes(containerAttributes) + .disruptionBudget(disruptionBudget) + .serviceJobProcesses(serviceJobProcesses); if (!securityGroups.isEmpty()) { - submitJobRequest.withSecurityGroups(securityGroups); + submitJobRequest.securityGroups(securityGroups); } if (dockerImage.getImageDigest() != null) { - submitJobRequest.withDockerDigest(dockerImage.getImageDigest()); + submitJobRequest.dockerDigest(dockerImage.getImageDigest()); } else { - submitJobRequest.withDockerImageVersion(dockerImage.getImageVersion()); + submitJobRequest.dockerImageVersion(dockerImage.getImageVersion()); } /** @@ -130,24 +140,25 @@ public SubmitJobRequest toSubmitJobRequest(@Nonnull DockerImage dockerImage) { * list */ if (constraints.getHard() != null || constraints.getSoft() != null) { - submitJobRequest.withConstraints(constraints); + submitJobRequest.containerConstraints(constraints); } else { log.warn("Use of deprecated constraints payload: {}-{}", application, stack); + + List constraints = new ArrayList<>(); if (hardConstraints != null) { - hardConstraints.forEach( - c -> submitJobRequest.withConstraint(SubmitJobRequest.Constraint.hard(c))); + hardConstraints.forEach(c -> constraints.add(SubmitJobRequest.Constraint.hard(c))); } if (softConstraints != null) { - softConstraints.forEach( - c -> submitJobRequest.withConstraint(SubmitJobRequest.Constraint.soft(c))); + softConstraints.forEach(c -> constraints.add(SubmitJobRequest.Constraint.soft(c))); } + submitJobRequest.constraints(constraints); } if (jobType != null) { - submitJobRequest.withJobType(jobType); + submitJobRequest.jobType(jobType); } - return submitJobRequest; + return submitJobRequest.build(); } @Data diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusJobSubmitted.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusJobSubmitted.java index 3a70685beb2..040159aeaa6 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusJobSubmitted.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusJobSubmitted.java @@ -15,26 +15,28 @@ */ package com.netflix.spinnaker.clouddriver.titus.deploy.events; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.netflix.spinnaker.clouddriver.saga.SagaEvent; import com.netflix.spinnaker.clouddriver.titus.JobType; import java.util.Map; import javax.annotation.Nonnull; -import lombok.Getter; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Value; -@Getter +@Builder(builderClassName = "TitusJobSubmittedBuilder", toBuilder = true) +@JsonDeserialize(builder = TitusJobSubmitted.TitusJobSubmittedBuilder.class) +@JsonTypeName("titusJobSubmitted") +@Value +@EqualsAndHashCode(callSuper = true) public class TitusJobSubmitted extends SagaEvent { @Nonnull private final Map serverGroupNameByRegion; @Nonnull private final String jobUri; @Nonnull private final JobType jobType; - public TitusJobSubmitted( - @Nonnull Map serverGroupNameByRegion, - @Nonnull String jobUri, - @Nonnull JobType jobType) { - super(); - this.serverGroupNameByRegion = serverGroupNameByRegion; - this.jobUri = jobUri; - this.jobType = jobType; - } + @JsonPOJOBuilder(withPrefix = "") + public static class TitusJobSubmittedBuilder {} } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusLoadBalancerAttached.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusLoadBalancerAttached.java index b7ce2363718..8e3948729df 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusLoadBalancerAttached.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusLoadBalancerAttached.java @@ -15,19 +15,25 @@ */ package com.netflix.spinnaker.clouddriver.titus.deploy.events; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.netflix.spinnaker.clouddriver.saga.SagaEvent; import javax.annotation.Nonnull; -import lombok.Getter; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Value; -@Getter +@Builder(builderClassName = "TitusLoadBalancerAttachedBuilder", toBuilder = true) +@JsonDeserialize(builder = TitusLoadBalancerAttached.TitusLoadBalancerAttachedBuilder.class) +@JsonTypeName("titusLoadBalancerAttached") +@Value +@EqualsAndHashCode(callSuper = true) public class TitusLoadBalancerAttached extends SagaEvent { @Nonnull private final String jobUri; @Nonnull private final String targetGroupArn; - public TitusLoadBalancerAttached(@Nonnull String jobUri, @Nonnull String targetGroupArn) { - super(); - this.jobUri = jobUri; - this.targetGroupArn = targetGroupArn; - } + @JsonPOJOBuilder(withPrefix = "") + public static class TitusLoadBalancerAttachedBuilder {} } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusScalingPolicyCopied.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusScalingPolicyCopied.java index f8e576f387f..54ab81e4e81 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusScalingPolicyCopied.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/events/TitusScalingPolicyCopied.java @@ -15,21 +15,26 @@ */ package com.netflix.spinnaker.clouddriver.titus.deploy.events; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.netflix.spinnaker.clouddriver.saga.SagaEvent; import javax.annotation.Nonnull; -import lombok.Getter; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Value; -@Getter +@Builder(builderClassName = "TitusScalingPolicyCopiedBuilder", toBuilder = true) +@JsonDeserialize(builder = TitusScalingPolicyCopied.TitusScalingPolicyCopiedBuilder.class) +@JsonTypeName("titusScalingPolicyCopied") +@Value +@EqualsAndHashCode(callSuper = true) public class TitusScalingPolicyCopied extends SagaEvent { + @Nonnull private final String serverGroupName; @Nonnull private final String region; @Nonnull private final String sourcePolicyId; - public TitusScalingPolicyCopied( - @Nonnull String serverGroupName, @Nonnull String region, @Nonnull String sourcePolicyId) { - super(); - this.serverGroupName = serverGroupName; - this.region = region; - this.sourcePolicyId = sourcePolicyId; - } + @JsonPOJOBuilder(withPrefix = "") + public static class TitusScalingPolicyCopiedBuilder {} } diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/TitusDeployHandler.java b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/TitusDeployHandler.java index 97967bdde05..1a3ff919e56 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/TitusDeployHandler.java +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/TitusDeployHandler.java @@ -64,10 +64,12 @@ public TitusDeploymentResult handle( sagaName, sagaId, flow, - new LoadFront50AppCommand( - inputDescription.getApplication(), - new PrepareTitusDeployCommand(inputDescription), - true)); + LoadFront50AppCommand.builder() + .appName(inputDescription.getApplication()) + .nextCommand( + PrepareTitusDeployCommand.builder().description(inputDescription).build()) + .allowMissing(true) + .build()); if (result == null) { // "This should never happen" diff --git a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/config/TitusConfiguration.groovy b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/config/TitusConfiguration.groovy index bb32e204c8e..689deacb44b 100644 --- a/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/config/TitusConfiguration.groovy +++ b/clouddriver-titus/src/main/groovy/com/netflix/spinnaker/config/TitusConfiguration.groovy @@ -17,6 +17,7 @@ package com.netflix.spinnaker.config import com.netflix.spectator.api.Registry +import com.netflix.spinnaker.clouddriver.event.SpinnakerEvent import com.netflix.spinnaker.clouddriver.saga.config.SagaAutoConfiguration import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository import com.netflix.spinnaker.clouddriver.titus.TitusClientProvider @@ -26,6 +27,8 @@ import com.netflix.spinnaker.clouddriver.titus.client.TitusRegion import com.netflix.spinnaker.clouddriver.titus.client.model.GrpcChannelFactory import com.netflix.spinnaker.clouddriver.titus.credentials.NetflixTitusCredentials import com.netflix.spinnaker.kork.core.RetrySupport +import com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer +import com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer.SubtypeLocator import groovy.util.logging.Slf4j import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty @@ -115,4 +118,12 @@ class TitusConfiguration { List featureFlags } } + + @Bean + SubtypeLocator titusEventSubtypeLocator() { + return new ObjectMapperSubtypeConfigurer.ClassSubtypeLocator( + SpinnakerEvent.class, + Collections.singletonList("com.netflix.spinnaker.clouddriver.titus") + ); + } } diff --git a/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/descriptions/TitusDeployDescriptionSpec.groovy b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/descriptions/TitusDeployDescriptionSpec.groovy new file mode 100644 index 00000000000..3091ae1b536 --- /dev/null +++ b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/descriptions/TitusDeployDescriptionSpec.groovy @@ -0,0 +1,79 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.titus.deploy.descriptions + +import com.fasterxml.jackson.databind.ObjectMapper +import com.netflix.spinnaker.clouddriver.titus.client.model.MigrationPolicy +import com.netflix.spinnaker.clouddriver.titus.credentials.NetflixTitusCredentials +import com.netflix.spinnaker.clouddriver.titus.deploy.description.TitusDeployDescription +import spock.lang.Specification +import spock.lang.Unroll + +class TitusDeployDescriptionSpec extends Specification { + + @Unroll + def "ser/de"() { + given: + ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules() + + and: + TitusDeployDescription subject = new TitusDeployDescription( + account: "titustest", + region: "us-east-1", + application: "helloworld", + capacity: new TitusDeployDescription.Capacity( + desired: 1, + max: 1, + min: 1 + ), + capacityGroup: "helloworld", + containerAttributes: [:], + credentials: credentials, + env: [:], + hardConstraints: [], + iamProfile: "helloworldInstanceProfile", + imageId: "titus/helloworld:latest", + inService: true, + labels: [:], + migrationPolicy: new MigrationPolicy( + type: "systemDefault" + ), + resources: new TitusDeployDescription.Resources( + allocateIpAddress: true, + cpu: 2, + disk: 10000, + memory: 4096, + networkMbps: 128 + ), + securityGroups: [], + softConstraints: [] + ) + + when: + objectMapper.readValue(objectMapper.writeValueAsString(subject), TitusDeployDescription) + + then: + noExceptionThrown() + + where: + credentials << [ + null, + Mock(NetflixTitusCredentials) { + getName() >> "titustest" + } + ] + } +} diff --git a/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/CommandSerdeSpec.groovy b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/CommandSerdeSpec.groovy new file mode 100644 index 00000000000..d584e0b6a8d --- /dev/null +++ b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/CommandSerdeSpec.groovy @@ -0,0 +1,124 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.clouddriver.titus.deploy.handlers.actions + +import com.fasterxml.jackson.databind.ObjectMapper +import com.netflix.spinnaker.clouddriver.event.CompositeSpinnakerEvent +import com.netflix.spinnaker.clouddriver.event.EventMetadata +import com.netflix.spinnaker.clouddriver.event.SpinnakerEvent +import com.netflix.spinnaker.clouddriver.titus.client.model.SubmitJobRequest +import com.netflix.spinnaker.clouddriver.titus.credentials.NetflixTitusCredentials +import com.netflix.spinnaker.clouddriver.titus.deploy.actions.SubmitTitusJob +import com.netflix.spinnaker.clouddriver.titus.deploy.description.TitusDeployDescription +import spock.lang.Shared +import spock.lang.Specification +import spock.lang.Unroll + +import java.time.Instant + +import static com.netflix.spinnaker.clouddriver.aws.deploy.ops.loadbalancer.TargetGroupLookupHelper.TargetGroupLookupResult +import static com.netflix.spinnaker.clouddriver.titus.deploy.actions.AttachTitusServiceLoadBalancers.AttachTitusServiceLoadBalancersCommand +import static com.netflix.spinnaker.clouddriver.titus.deploy.actions.CopyTitusServiceScalingPolicies.CopyTitusServiceScalingPoliciesCommand +import static com.netflix.spinnaker.clouddriver.titus.deploy.actions.LoadFront50App.Front50App +import static com.netflix.spinnaker.clouddriver.titus.deploy.actions.LoadFront50App.LoadFront50AppCommand +import static com.netflix.spinnaker.clouddriver.titus.deploy.actions.PrepareTitusDeploy.PrepareTitusDeployCommand + +class CommandSerdeSpec extends Specification { + + @Shared NetflixTitusCredentials titusCredentials = Mock() { + getName() >> "titus" + } + + @Shared TitusDeployDescription deployDescription = new TitusDeployDescription(credentials: titusCredentials) + + @Shared Front50App front50App = new Front50App("example@example.com", true) + + @Unroll + def "can serialize and deserialize #command.class.simpleName"() { + given: + ObjectMapper objectMapper = new ObjectMapper() + objectMapper + .findAndRegisterModules() + + and: + registerSubtypes(objectMapper, command) + initializeEvent(command) + + when: + def serialized = objectMapper.writeValueAsString(command) + objectMapper.readValue(serialized, SpinnakerEvent) + + then: + noExceptionThrown() + + where: + command << [ + LoadFront50AppCommand.builder() + .appName("myApp") + .nextCommand(PrepareTitusDeployCommand.builder() + .description(deployDescription) + .front50App(front50App) + .build()) + .allowMissing(true) + .build(), + PrepareTitusDeployCommand.builder() + .description(deployDescription) + .front50App(front50App) + .build(), + AttachTitusServiceLoadBalancersCommand.builder() + .description(deployDescription) + .jobUri("http://localhost/id") + .targetGroupLookupResult(new TargetGroupLookupResult()) + .build(), + CopyTitusServiceScalingPoliciesCommand.builder() + .description(deployDescription) + .jobUri("http://localhost/id") + .deployedServerGroupName("myapp-v000") + .build(), + SubmitTitusJob.SubmitTitusJobCommand.builder() + .description(deployDescription) + .submitJobRequest(SubmitJobRequest.builder().build()) + .nextServerGroupName("myapp-v000") + .front50App(front50App) + .build() + ] + } + + static void registerSubtypes(ObjectMapper objectMapper, SpinnakerEvent event) { + if (event instanceof CompositeSpinnakerEvent) { + objectMapper.registerSubtypes(((CompositeSpinnakerEvent) event).composedEvents.collect { it.class }) + } + objectMapper.registerSubtypes(event.class) + } + + static void initializeEvent(SpinnakerEvent event) { + if (event instanceof CompositeSpinnakerEvent) { + event.composedEvents.forEach { + initializeEvent(it) + } + } + event.metadata = new EventMetadata( + UUID.randomUUID().toString(), + "aggType", + "aggId", + 0, + 0, + Instant.now(), + "unknown", + "unknown" + ) + } +} diff --git a/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/PrepareTitusDeployActionSpec.groovy b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/PrepareTitusDeployActionSpec.groovy index 5c8d87818f3..04967158f8c 100644 --- a/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/PrepareTitusDeployActionSpec.groovy +++ b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/PrepareTitusDeployActionSpec.groovy @@ -222,9 +222,9 @@ class PrepareTitusDeployActionSpec extends Specification { private static PrepareTitusDeployCommand createCommand( TitusDeployDescription description, String email, boolean platformHealthOnly) { - return new PrepareTitusDeployCommand(description).with { - it.front50App = new LoadFront50App.Front50App(email, platformHealthOnly) - it - } + return PrepareTitusDeployCommand.builder() + .description(description) + .front50App(new LoadFront50App.Front50App(email, platformHealthOnly)) + .build() } } diff --git a/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/TitusJobNameResolverSpec.groovy b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/TitusJobNameResolverSpec.groovy index 8bcadab850a..9b00a0bcce2 100644 --- a/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/TitusJobNameResolverSpec.groovy +++ b/clouddriver-titus/src/test/groovy/com/netflix/spinnaker/clouddriver/titus/deploy/handlers/actions/TitusJobNameResolverSpec.groovy @@ -20,7 +20,6 @@ import com.netflix.spinnaker.clouddriver.data.task.TaskRepository import com.netflix.spinnaker.clouddriver.titus.JobType import com.netflix.spinnaker.clouddriver.titus.client.TitusClient import com.netflix.spinnaker.clouddriver.titus.client.model.Job -import com.netflix.spinnaker.clouddriver.titus.client.model.SubmitJobRequest import com.netflix.spinnaker.clouddriver.titus.deploy.actions.TitusJobNameResolver import com.netflix.spinnaker.clouddriver.titus.deploy.description.TitusDeployDescription import spock.lang.Specification @@ -51,13 +50,9 @@ class TitusJobNameResolverSpec extends Specification { ] } } - SubmitJobRequest submitJobRequest = new SubmitJobRequest().with { - it.jobType = description.jobType - it - } when: - String result = TitusJobNameResolver.resolveJobName(titusClient, description, submitJobRequest) + String result = TitusJobNameResolver.resolveJobName(titusClient, description) then: result == expected