feat(provider/ecs): ECS Service caching classes and tests. (#2144)

* ECS Service caching classes and tests.

* Objectmapper is now injected.

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/ServiceCacheClient.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.client;
+
+import com.amazonaws.services.ecs.model.LoadBalancer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.spinnaker.cats.cache.Cache;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
+
+public class ServiceCacheClient extends AbstractCacheClient<Service> {
+  private ObjectMapper mapper;
+
+  @Autowired
+  public ServiceCacheClient(Cache cacheView, ObjectMapper mapper) {
+    super(cacheView, SERVICES.toString());
+    this.mapper = mapper;
+  }
+
+  @Override
+  protected Service convert(CacheData cacheData) {
+    Service service = new Service();
+    Map<String, Object> attributes = cacheData.getAttributes();
+
+    service.setAccount((String) attributes.get("account"));
+    service.setRegion((String) attributes.get("region"));
+    service.setApplicationName((String) attributes.get("applicationName"));
+    service.setServiceName((String) attributes.get("serviceName"));
+    service.setServiceArn((String) attributes.get("serviceArn"));
+    service.setClusterName((String) attributes.get("clusterName"));
+    service.setClusterArn((String) attributes.get("clusterArn"));
+    service.setRoleArn((String) attributes.get("roleArn"));
+    service.setTaskDefinition((String) attributes.get("taskDefinition"));
+    service.setDesiredCount((Integer) attributes.get("desiredCount"));
+    service.setMaximumPercent((Integer) attributes.get("maximumPercent"));
+    service.setMinimumHealthyPercent((Integer) attributes.get("minimumHealthyPercent"));
+
+    if (attributes.containsKey("loadBalancers")) {
+      List<Map<String, Object>> loadBalancers = (List<Map<String, Object>>) attributes.get("loadBalancers");
+      List<LoadBalancer> deserializedLoadbalancers = new ArrayList<>(loadBalancers.size());
+
+      for (Map<String, Object> serializedLoadbalancer : loadBalancers) {
+        if(serializedLoadbalancer!=null) {
+          deserializedLoadbalancers.add(mapper.convertValue(serializedLoadbalancer, LoadBalancer.class));
+        }
+      }
+
+      service.setLoadBalancers(deserializedLoadbalancers);
+    } else {
+      service.setLoadBalancers(Collections.emptyList());
+    }
+
+
+    service.setCreatedAt((Long) attributes.get("createdAt"));
+
+    return service;
+  }
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/model/Service.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.model;
+
+import com.amazonaws.services.ecs.model.LoadBalancer;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class Service {
+  String account;
+  String region;
+  String applicationName;
+  String serviceName;
+  String serviceArn;
+  String clusterName;
+  String clusterArn;
+  String roleArn;
+  String taskDefinition;
+  int desiredCount;
+  int maximumPercent;
+  int minimumHealthyPercent;
+  List<LoadBalancer> loadBalancers;
+  long createdAt;
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/ServiceCachingAgent.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.ecs.AmazonECS;
+import com.amazonaws.services.ecs.model.DescribeServicesRequest;
+import com.amazonaws.services.ecs.model.ListServicesRequest;
+import com.amazonaws.services.ecs.model.ListServicesResult;
+import com.amazonaws.services.ecs.model.Service;
+import com.netflix.spectator.api.Registry;
+import com.netflix.spinnaker.cats.agent.AgentDataType;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
+import com.netflix.spinnaker.cats.provider.ProviderCache;
+import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
+import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
+import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.INFORMATIVE;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
+
+public class ServiceCachingAgent extends AbstractEcsOnDemandAgent<Service> {
+  private static final Collection<AgentDataType> types = Collections.unmodifiableCollection(Arrays.asList(
+    AUTHORITATIVE.forType(SERVICES.toString()),
+    INFORMATIVE.forType(ECS_CLUSTERS.toString())
+  ));
+  private final Logger log = LoggerFactory.getLogger(getClass());
+
+  public ServiceCachingAgent(String accountName, String region, AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry) {
+    super(accountName, region, amazonClientProvider, awsCredentialsProvider, registry);
+  }
+
+  public static Map<String, Object> convertServiceToAttributes(String accountName, String region, Service service) {
+    Map<String, Object> attributes = new HashMap<>();
+    String applicationName = service.getServiceName().contains("-") ? StringUtils.substringBefore(service.getServiceName(), "-") : service.getServiceName();
+    String clusterName = StringUtils.substringAfterLast(service.getClusterArn(), "/");
+
+    attributes.put("account", accountName);
+    attributes.put("region", region);
+    attributes.put("applicationName", applicationName);
+    attributes.put("serviceName", service.getServiceName());
+    attributes.put("serviceArn", service.getServiceArn());
+    attributes.put("clusterName", clusterName);
+    attributes.put("clusterArn", service.getClusterArn());
+    attributes.put("roleArn", service.getRoleArn());
+    attributes.put("taskDefinition", service.getTaskDefinition());
+    attributes.put("desiredCount", service.getDesiredCount());
+    attributes.put("maximumPercent", service.getDeploymentConfiguration().getMaximumPercent());
+    attributes.put("minimumHealthyPercent", service.getDeploymentConfiguration().getMinimumHealthyPercent());
+    attributes.put("loadBalancers", service.getLoadBalancers());
+    attributes.put("createdAt", service.getCreatedAt().getTime());
+
+    return attributes;
+  }
+
+  @Override
+  public String getAgentType() {
+    return ServiceCachingAgent.class.getSimpleName();
+  }
+
+  @Override
+  public Collection<AgentDataType> getProvidedDataTypes() {
+    return types;
+  }
+
+  @Override
+  protected List<Service> getItems(AmazonECS ecs, ProviderCache providerCache) {
+    List<Service> serviceList = new LinkedList<>();
+    Set<String> clusters = getClusters(ecs, providerCache);
+
+    for (String cluster : clusters) {
+      String nextToken = null;
+      do {
+        ListServicesRequest listServicesRequest = new ListServicesRequest().withCluster(cluster);
+        if (nextToken != null) {
+          listServicesRequest.setNextToken(nextToken);
+        }
+        ListServicesResult listServicesResult = ecs.listServices(listServicesRequest);
+        List<String> serviceArns = listServicesResult.getServiceArns();
+        if (serviceArns.size() == 0) {
+          continue;
+        }
+
+        List<Service> services = ecs.describeServices(new DescribeServicesRequest().withCluster(cluster).withServices(serviceArns)).getServices();
+        serviceList.addAll(services);
+
+        nextToken = listServicesResult.getNextToken();
+      } while (nextToken != null && nextToken.length() != 0);
+    }
+    return serviceList;
+  }
+
+  @Override
+  protected Map<String, Collection<CacheData>> generateFreshData(Collection<Service> services) {
+    Collection<CacheData> dataPoints = new LinkedList<>();
+    Map<String, CacheData> clusterDataPoints = new HashMap<>();
+
+    for (Service service : services) {
+      Map<String, Object> attributes = convertServiceToAttributes(accountName, region, service);
+
+      String key = Keys.getServiceKey(accountName, region, service.getServiceName());
+      dataPoints.add(new DefaultCacheData(key, attributes, Collections.emptyMap()));
+
+      Map<String, Object> clusterAttributes = EcsClusterCachingAgent.convertClusterArnToAttributes(accountName, region, service.getClusterArn());
+      String clusterName = StringUtils.substringAfterLast(service.getClusterArn(), "/");
+      key = Keys.getClusterKey(accountName, region, clusterName);
+      clusterDataPoints.put(key, new DefaultCacheData(key, clusterAttributes, Collections.emptyMap()));
+    }
+
+    log.info("Caching " + dataPoints.size() + " services in " + getAgentType());
+    Map<String, Collection<CacheData>> dataMap = new HashMap<>();
+    dataMap.put(SERVICES.toString(), dataPoints);
+
+    log.info("Caching " + clusterDataPoints.size() + " ECS clusters in " + getAgentType());
+    dataMap.put(ECS_CLUSTERS.toString(), clusterDataPoints.values());
+
+    return dataMap;
+  }
+}

clouddriver-ecs/src/test/java/com/netflix/spinnaker/clouddriver/ecs/cache/ServiceCacheClientTest.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache;
+
+import com.amazonaws.services.ecs.model.DeploymentConfiguration;
+import com.amazonaws.services.ecs.model.LoadBalancer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.ServiceCacheClient;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.ServiceCachingAgent;
+import org.junit.Test;
+import spock.lang.Subject;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.Map;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+public class ServiceCacheClientTest extends CommonCacheClient {
+  private final ObjectMapper mapper = new ObjectMapper();
+  @Subject
+  private final ServiceCacheClient client = new ServiceCacheClient(cacheView, mapper);
+
+  @Test
+  public void shouldConvert() {
+    //Given
+    ObjectMapper mapper = new ObjectMapper();
+    String applicationName = "test";
+    String serviceName = applicationName + "-stack-detail-v1";
+    String key = Keys.getServiceKey(ACCOUNT, REGION, serviceName);
+    String clusterName = "test-cluster";
+
+    LoadBalancer loadBalancer = new LoadBalancer();
+    loadBalancer.setContainerName("container-name");
+    loadBalancer.setContainerPort(8080);
+    loadBalancer.setLoadBalancerName("balancer-of-load");
+    loadBalancer.setTargetGroupArn("target-group-arn");
+
+    com.amazonaws.services.ecs.model.Service service = new com.amazonaws.services.ecs.model.Service();
+    service.setServiceName(serviceName);
+    service.setServiceArn("arn:aws:ecs:" + REGION + ":012345678910:service/" + serviceName);
+    service.setClusterArn("arn:aws:ecs:" + REGION + ":012345678910:cluster/" + clusterName);
+    service.setTaskDefinition("arn:aws:ecs:" + REGION + ":012345678910:task-definition/test-task-def:1");
+    service.setRoleArn("arn:aws:ecs:" + REGION + ":012345678910:service/test-role");
+    service.setDeploymentConfiguration(new DeploymentConfiguration().withMinimumHealthyPercent(50).withMaximumPercent(100));
+    service.setLoadBalancers(Collections.singleton(loadBalancer));
+    service.setDesiredCount(9001);
+    service.setCreatedAt(new Date());
+    Map<String, Object> attributes = ServiceCachingAgent.convertServiceToAttributes(ACCOUNT, REGION, service);
+    attributes.put("loadBalancers", Collections.singletonList(mapper.convertValue(loadBalancer, Map.class)));
+
+    when(cacheView.get(SERVICES.toString(), key)).thenReturn(new DefaultCacheData(key, attributes, Collections.emptyMap()));
+
+    //When
+    Service ecsService = client.get(key);
+
+    //Then
+    assertTrue("Expected the cluster name to be " + clusterName + " but got " + ecsService.getClusterName(),
+      clusterName.equals(ecsService.getClusterName()));
+
+    assertTrue("Expected the cluster ARN to be " + service.getClusterArn() + " but got " + ecsService.getClusterArn(),
+      service.getClusterArn().equals(ecsService.getClusterArn()));
+
+    assertTrue("Expected the account of the service to be " + ACCOUNT + " but got " + ecsService.getAccount(),
+      ACCOUNT.equals(ecsService.getAccount()));
+
+    assertTrue("Expected the region of the service to be " + REGION + " but got " + ecsService.getRegion(),
+      REGION.equals(ecsService.getRegion()));
+
+    assertTrue("Expected the service application name to be " + applicationName + " but got " + ecsService.getApplicationName(),
+      applicationName.equals(ecsService.getApplicationName()));
+
+    assertTrue("Expected the service name to be " + serviceName + " but got " + ecsService.getServiceName(),
+      serviceName.equals(ecsService.getServiceName()));
+
+    assertTrue("Expected the service ARN to be " + service.getServiceArn() + " but got " + ecsService.getServiceArn(),
+      service.getServiceArn().equals(ecsService.getServiceArn()));
+
+    assertTrue("Expected the role ARN of the service to be " + service.getRoleArn() + " but got " + ecsService.getRoleArn(),
+      service.getRoleArn().equals(ecsService.getRoleArn()));
+
+    assertTrue("Expected the task definition of the service to be " + service.getTaskDefinition() + " but got " + ecsService.getTaskDefinition(),
+      service.getTaskDefinition().equals(ecsService.getTaskDefinition()));
+
+    assertTrue("Expected the desired count of the service to be " + service.getDesiredCount() + " but got " + ecsService.getDesiredCount(),
+      service.getDesiredCount() == ecsService.getDesiredCount());
+
+    assertTrue("Expected the maximum percent of the service to be " + service.getDeploymentConfiguration().getMaximumPercent() + " but got " + ecsService.getMaximumPercent(),
+      service.getDeploymentConfiguration().getMaximumPercent() == ecsService.getMaximumPercent());
+
+    assertTrue("Expected the minimum healthy percent of the service to be " + service.getDeploymentConfiguration().getMinimumHealthyPercent() + " but got " + ecsService.getMinimumHealthyPercent(),
+      service.getDeploymentConfiguration().getMinimumHealthyPercent() == ecsService.getMinimumHealthyPercent());
+
+    assertTrue("Expected the created at of the service to be " + service.getCreatedAt().getTime() + " but got " + ecsService.getCreatedAt(),
+      service.getCreatedAt().getTime() == ecsService.getCreatedAt());
+
+    assertTrue("Expected the service to have 1 load balancer but got " + ecsService.getLoadBalancers().size(),
+      ecsService.getLoadBalancers().size() == 1);
+
+    assertTrue("Expected the service to have load balancer " + loadBalancer + " but got " + ecsService.getLoadBalancers().get(0),
+      ecsService.getLoadBalancers().get(0).equals(loadBalancer));
+  }
+}