Skip to content

Commit

Permalink
implement autoscaling crud for titus
Browse files Browse the repository at this point in the history
  • Loading branch information
anotherchrisberry authored and tomaslin committed Mar 26, 2018
1 parent dd2b91e commit 8ae2092
Show file tree
Hide file tree
Showing 15 changed files with 697 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
package com.netflix.spinnaker.clouddriver.titus

import com.netflix.spectator.api.Registry
import com.netflix.spinnaker.clouddriver.titus.client.RegionScopedTitusAutoscalingClient
import com.netflix.spinnaker.clouddriver.titus.client.RegionScopedTitusClient
import com.netflix.spinnaker.clouddriver.titus.client.TitusAutoscalingClient
import com.netflix.spinnaker.clouddriver.titus.client.TitusJobCustomizer
import com.netflix.spinnaker.clouddriver.titus.client.TitusRegion
import com.netflix.spinnaker.clouddriver.titus.credentials.NetflixTitusCredentials
Expand All @@ -30,6 +32,7 @@ import java.util.concurrent.ConcurrentHashMap
class TitusClientProvider {

private final Map<TitusClientKey, TitusClient> titusClients = new ConcurrentHashMap<>()
private final Map<TitusClientKey, TitusAutoscalingClient> titusAutoscalingClients = new ConcurrentHashMap<>()
private final Registry registry
private final List<TitusJobCustomizer> titusJobCustomizers

Expand All @@ -44,6 +47,15 @@ class TitusClientProvider {
return titusClients.computeIfAbsent(key, { k -> k.region.apiVersion == '3' ? new RegionScopedV3TitusClient(k.region, registry, titusJobCustomizers, account.environment, account.eurekaName) : new RegionScopedTitusClient(k.region, registry, titusJobCustomizers) })
}

TitusAutoscalingClient getTitusAutoscalingClient(NetflixTitusCredentials account, String region) {
final TitusRegion titusRegion = Objects.requireNonNull(account.regions.find { it.name == region }, "region")
if (titusRegion.apiVersion != '3') {
return null
}
final TitusClientKey key = new TitusClientKey(Objects.requireNonNull(account.name), titusRegion)
return titusAutoscalingClients.computeIfAbsent(key, { k -> new RegionScopedTitusAutoscalingClient(k.region, registry, account.environment, account.eurekaName) })
}

@Immutable(knownImmutableClasses = [TitusRegion])
static class TitusClientKey {
final String account
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package com.netflix.spinnaker.clouddriver.titus.caching.agents
import com.fasterxml.jackson.annotation.JsonCreator
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.databind.ObjectMapper
import com.google.protobuf.util.JsonFormat
import com.netflix.frigga.Names
import com.netflix.frigga.autoscaling.AutoScalingGroupNameBuilder
import com.netflix.spinnaker.cats.agent.AgentDataType
Expand All @@ -33,11 +34,15 @@ import com.netflix.spinnaker.clouddriver.titus.TitusClientProvider
import com.netflix.spinnaker.clouddriver.titus.caching.Keys
import com.netflix.spinnaker.clouddriver.titus.caching.TitusCachingProvider
import com.netflix.spinnaker.clouddriver.titus.caching.utils.AwsLookupUtil
import com.netflix.spinnaker.clouddriver.titus.client.TitusAutoscalingClient
import com.netflix.spinnaker.clouddriver.titus.client.TitusClient
import com.netflix.spinnaker.clouddriver.titus.client.model.Job
import com.netflix.spinnaker.clouddriver.titus.client.model.TaskState
import com.netflix.spinnaker.clouddriver.titus.credentials.NetflixTitusCredentials
import com.netflix.spinnaker.clouddriver.titus.model.TitusSecurityGroup
import com.netflix.titus.grpc.protogen.ScalingPolicy
import com.netflix.titus.grpc.protogen.ScalingPolicyResult
import com.netflix.titus.grpc.protogen.ScalingPolicyStatus.ScalingPolicyState
import org.slf4j.Logger
import org.slf4j.LoggerFactory

Expand All @@ -60,6 +65,7 @@ class TitusClusterCachingAgent implements CachingAgent, CustomScheduledAgent {
] as Set)

private final TitusClient titusClient
private final TitusAutoscalingClient titusAutoscalingClient
private final NetflixTitusCredentials account
private final String region
private final ObjectMapper objectMapper
Expand All @@ -79,6 +85,7 @@ class TitusClusterCachingAgent implements CachingAgent, CustomScheduledAgent {
this.region = region
this.objectMapper = objectMapper
this.titusClient = titusClientProvider.getTitusClient(account, region)
this.titusAutoscalingClient = titusClientProvider.getTitusAutoscalingClient(account, region)
this.awsLookupUtil = awsLookupUtil
this.pollIntervalMillis = pollIntervalMillis
this.timeoutMillis = timeoutMillis
Expand Down Expand Up @@ -134,18 +141,25 @@ class TitusClusterCachingAgent implements CachingAgent, CustomScheduledAgent {
Map<String, CacheData> clusters = createCache()
Map<String, CacheData> serverGroups = createCache()
Map<String, CacheData> instances = createCache()

List<ScalingPolicyResult> allScalingPolicies = titusAutoscalingClient ? titusAutoscalingClient.getAllScalingPolicies() : []
// Ignore policies in a Deleted state (may need to revisit)
List cacheablePolicyStates = [ScalingPolicyState.Pending, ScalingPolicyState.Applied, ScalingPolicyState.Deleting]
Map<String, TitusSecurityGroup> titusSecurityGroupCache = [:]

for (Job job : jobs) {
try {
ServerGroupData data = new ServerGroupData(job, account.name, region, account.stack)
cacheApplication(data, applications)
cacheCluster(data, clusters)
cacheServerGroup(data, serverGroups, instances, titusSecurityGroupCache)
} catch (Exception ex) {
log.error("Failed to cache ${job.name} in ${account.name}", ex)
try {
List<ScalingPolicy> scalingPolicies = allScalingPolicies.findResults {
it.jobId == job.id && cacheablePolicyStates.contains(it.policyState.state) ?
it.scalingPolicy :
null
}
ServerGroupData data = new ServerGroupData(job, scalingPolicies, account.name, region, account.stack)
cacheApplication(data, applications)
cacheCluster(data, clusters)
cacheServerGroup(data, serverGroups, instances, titusSecurityGroupCache)
} catch (Exception ex) {
log.error("Failed to cache ${job.name} in ${account.name}", ex)
}
}

new DefaultCacheResult(
Expand Down Expand Up @@ -178,7 +192,13 @@ class TitusClusterCachingAgent implements CachingAgent, CustomScheduledAgent {
serverGroups[data.serverGroup].with {
Job job = objectMapper.convertValue(data.job, Job.class)
resolveAwsDetails(titusSecurityGroupCache, job)
List<Map> policies = data.scalingPolicies ? data.scalingPolicies.collect {
// There is probably a better way to convert a protobuf to a Map, but I don't know what it is
objectMapper.readValue(JsonFormat.printer().print(it), Map)
} : []

attributes.job = job
attributes.scalingPolicies = policies
attributes.tasks = data.job.tasks
attributes.region = region
attributes.account = account.name
Expand Down Expand Up @@ -212,6 +232,7 @@ class TitusClusterCachingAgent implements CachingAgent, CustomScheduledAgent {
private class ServerGroupData {

final Job job
List<ScalingPolicy> scalingPolicies
final Names name
final String appName
final String cluster
Expand All @@ -220,8 +241,9 @@ class TitusClusterCachingAgent implements CachingAgent, CustomScheduledAgent {
final String region
final String account

public ServerGroupData(Job job, String account, String region, String stack) {
ServerGroupData(Job job, List<ScalingPolicy> scalingPolicies, String account, String region, String stack) {
this.job = job
this.scalingPolicies = scalingPolicies

String asgName = job.name
if (job.labels && job.labels['name']) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ class TitusClusterProvider implements ClusterProvider<TitusCluster> {
TitusServerGroup serverGroup = new TitusServerGroup(job, serverGroupData.attributes.account, serverGroupData.attributes.region)
serverGroup.placement.account = account
serverGroup.placement.region = region
serverGroup.scalingPolicies = serverGroupData.attributes.scalingPolicies
serverGroup.instances = translateInstances(resolveRelationshipData(serverGroupData, INSTANCES.ns)).values()
serverGroup
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.clouddriver.titus.client;

import com.google.protobuf.Empty;
import com.netflix.eureka2.grpc.nameresolver.Eureka2NameResolverFactory;
import com.netflix.spectator.api.Registry;
import com.netflix.spinnaker.clouddriver.titus.v3client.ClientAuthenticationUtils;
import com.netflix.spinnaker.clouddriver.titus.v3client.GrpcMetricsInterceptor;
import com.netflix.spinnaker.clouddriver.titus.v3client.GrpcRetryInterceptor;
import com.netflix.titus.grpc.protogen.*;
import io.grpc.ManagedChannel;
import io.grpc.netty.NegotiationType;
import io.grpc.netty.NettyChannelBuilder;
import io.grpc.util.RoundRobinLoadBalancerFactory;

import java.util.List;

public class RegionScopedTitusAutoscalingClient implements TitusAutoscalingClient {

/**
* Default connect timeout in milliseconds
*/
private static final long DEFAULT_CONNECT_TIMEOUT = 5000;

private final AutoScalingServiceGrpc.AutoScalingServiceBlockingStub autoScalingServiceBlockingStub;

public RegionScopedTitusAutoscalingClient(TitusRegion titusRegion,
Registry registry,
String environment,
String eurekaName) {

ManagedChannel eurekaChannel = NettyChannelBuilder
.forTarget("eurekaproxy." + titusRegion.getName() + ".discovery" + environment + ".netflix.net:8980")
.loadBalancerFactory(RoundRobinLoadBalancerFactory.getInstance())
.usePlaintext(true)
.userAgent("spinnaker")
.build();

ManagedChannel channel = NettyChannelBuilder
.forTarget("eureka:///" + eurekaName + "?eureka.status=up")
.sslContext(ClientAuthenticationUtils.newSslContext("titusapi"))
.negotiationType(NegotiationType.TLS)
.nameResolverFactory(new Eureka2NameResolverFactory(eurekaChannel)) // This enables the client to resolve the Eureka URI above into a set of addressable service endpoints.
.loadBalancerFactory(RoundRobinLoadBalancerFactory.getInstance())
.intercept(new GrpcMetricsInterceptor(registry, titusRegion))
.intercept(new GrpcRetryInterceptor(DEFAULT_CONNECT_TIMEOUT))
.build();

this.autoScalingServiceBlockingStub = AutoScalingServiceGrpc.newBlockingStub(channel);
}

@Override
public List<ScalingPolicyResult> getAllScalingPolicies() {
return autoScalingServiceBlockingStub.getAllScalingPolicies(Empty.newBuilder().build()).getItemsList();
}

@Override
public List<ScalingPolicyResult> getJobScalingPolicies(String jobId) {
JobId request = JobId.newBuilder().setId(jobId).build();
return autoScalingServiceBlockingStub
.getJobScalingPolicies(request).getItemsList();
}

@Override
public ScalingPolicyID upsertScalingPolicy(PutPolicyRequest policy) {
return autoScalingServiceBlockingStub
.setAutoScalingPolicy(policy);
}

@Override
public void deleteScalingPolicy(DeletePolicyRequest request) {
autoScalingServiceBlockingStub.deleteAutoScalingPolicy(request);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.clouddriver.titus.client;

import com.netflix.titus.grpc.protogen.DeletePolicyRequest;
import com.netflix.titus.grpc.protogen.PutPolicyRequest;
import com.netflix.titus.grpc.protogen.ScalingPolicyID;
import com.netflix.titus.grpc.protogen.ScalingPolicyResult;

import java.util.List;

public interface TitusAutoscalingClient {

List<ScalingPolicyResult> getAllScalingPolicies();

List<ScalingPolicyResult> getJobScalingPolicies(String jobId);

ScalingPolicyID upsertScalingPolicy(PutPolicyRequest policy);

void deleteScalingPolicy(DeletePolicyRequest request);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.clouddriver.titus.deploy.converters

import com.netflix.spinnaker.clouddriver.orchestration.AtomicOperation
import com.netflix.spinnaker.clouddriver.orchestration.AtomicOperations
import com.netflix.spinnaker.clouddriver.security.AbstractAtomicOperationsCredentialsSupport
import com.netflix.spinnaker.clouddriver.titus.TitusOperation
import com.netflix.spinnaker.clouddriver.titus.deploy.description.DeleteTitusScalingPolicyDescription
import com.netflix.spinnaker.clouddriver.titus.deploy.ops.DeleteTitusScalingPolicyAtomicOperation
import org.springframework.stereotype.Component

@Component('titusDeleteScalingPolicyDescription')
@TitusOperation(AtomicOperations.DELETE_SCALING_POLICY)
class DeleteTitusScalingPolicyAtomicOperationConverter extends AbstractAtomicOperationsCredentialsSupport {

@Override
AtomicOperation convertOperation(Map input) {
new DeleteTitusScalingPolicyAtomicOperation(convertDescription(input))
}

@Override
DeleteTitusScalingPolicyDescription convertDescription(Map input) {
DeleteTitusScalingPolicyDescription converted = getObjectMapper().convertValue(input, DeleteTitusScalingPolicyDescription);
converted.credentials = getCredentialsObject(input.credentials as String)
converted
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.clouddriver.titus.deploy.converters

import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.netflix.spinnaker.clouddriver.aws.deploy.converters.UpsertScalingPolicyDescriptionAtomicOperationConverter
import com.netflix.spinnaker.clouddriver.aws.deploy.ops.UpsertScalingPolicyAtomicOperation
import com.netflix.spinnaker.clouddriver.orchestration.AtomicOperations
import com.netflix.spinnaker.clouddriver.security.AbstractAtomicOperationsCredentialsSupport
import com.netflix.spinnaker.clouddriver.titus.TitusOperation
import com.netflix.spinnaker.clouddriver.titus.deploy.description.UpsertTitusScalingPolicyDescription
import com.netflix.spinnaker.clouddriver.titus.deploy.ops.UpsertTitusScalingPolicyAtomicOperation
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component

@Component('titusUpsertScalingPolicyDescription')
@TitusOperation(AtomicOperations.UPSERT_SCALING_POLICY)
class UpsertTitusScalingPolicyAtomicOperationConverter extends AbstractAtomicOperationsCredentialsSupport {

@Autowired
ObjectMapper objectMapper

@Override
UpsertTitusScalingPolicyAtomicOperation convertOperation(Map input) {
new UpsertTitusScalingPolicyAtomicOperation(convertDescription(input))
}

@Override
UpsertTitusScalingPolicyDescription convertDescription(Map input) {
UpsertTitusScalingPolicyDescription converted = objectMapper.copy()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.convertValue(input, UpsertTitusScalingPolicyDescription)
converted.credentials = getCredentialsObject(input.credentials as String)
converted
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.clouddriver.titus.deploy.description

class DeleteTitusScalingPolicyDescription extends AbstractTitusCredentialsDescription {

String region
String scalingPolicyID
}
Loading

0 comments on commit 8ae2092

Please sign in to comment.