From b6bd26baaeb75f08c26bb2befc6fcf8dc8e5645f Mon Sep 17 00:00:00 2001 From: Michael Plump Date: Thu, 5 Sep 2019 14:04:41 -0400 Subject: [PATCH] refactor(gce): convert GoogleZonalServerGroupCachingAgent to Java (#4002) * refactor(naming): add type boundaries on NamerRegistry * refactor(gce): convert GoogleZonalServerGroupCachingAgent to Java * refactor(gce): address review comments * refactor(gce): remove static from a few methods so we can use fields * refactor(gce): Add @ParametersAreNonnullByDefault --- .../clouddriver/cache/OnDemandAgent.java | 2 + .../clouddriver/names/NamerRegistry.java | 21 +- .../clouddriver/google/cache/Keys.groovy | 3 + .../google/model/GoogleServerGroup.groovy | 2 +- ...ogleRegionalServerGroupCachingAgent.groovy | 189 +++- .../GoogleZonalServerGroupCachingAgent.groovy | 723 -------------- .../GoogleZonalServerGroupCachingAgent.java | 924 ++++++++++++++++++ .../GoogleInfrastructureProviderConfig.groovy | 17 +- .../GoogleServerGroupCachingAgentSpec.groovy | 142 --- ...oogleZonalServerGroupCachingAgentTest.java | 138 ++- .../provider/agent/StubComputeFactory.java | 62 +- 11 files changed, 1327 insertions(+), 896 deletions(-) delete mode 100644 clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.groovy create mode 100644 clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.java delete mode 100644 clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleServerGroupCachingAgentSpec.groovy diff --git a/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/cache/OnDemandAgent.java b/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/cache/OnDemandAgent.java index 135345f58e0..5d2b0578428 100644 --- a/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/cache/OnDemandAgent.java +++ b/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/cache/OnDemandAgent.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import javax.annotation.Nullable; import lombok.Data; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,6 +102,7 @@ default Moniker convertOnDemandDetails(Map details) { } } + @Nullable OnDemandResult handle(ProviderCache providerCache, Map data); Collection pendingOnDemandRequests(ProviderCache providerCache); diff --git a/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/names/NamerRegistry.java b/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/names/NamerRegistry.java index 3d4de94d5e4..1192c98401e 100644 --- a/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/names/NamerRegistry.java +++ b/clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/names/NamerRegistry.java @@ -30,12 +30,15 @@ * the mapping from (provider, account, resource) -< namer must happen within Spinnaker. */ public class NamerRegistry { - private final List namingStrategies; - private static Namer defaultNamer = new FriggaReflectiveNamer(); + + private static Namer DEFAULT_NAMER = new FriggaReflectiveNamer(); + private static ProviderLookup providerLookup = new ProviderLookup(); - public static Namer getDefaultNamer() { - return defaultNamer; + private final List namingStrategies; + + public static Namer getDefaultNamer() { + return DEFAULT_NAMER; } public static ProviderLookup lookup() { @@ -58,18 +61,18 @@ public Namer getNamingStrategy(String strategyName) { @Slf4j public static class ResourceLookup { - private ConcurrentHashMap map = new ConcurrentHashMap<>(); + private ConcurrentHashMap, Namer> map = new ConcurrentHashMap<>(); - public Namer withResource(Class resource) { + public Namer withResource(Class resource) { if (!map.containsKey(resource)) { log.debug("Looking up a namer for a non-registered resource"); - return getDefaultNamer(); + return (Namer) getDefaultNamer(); } else { - return map.get(resource); + return (Namer) map.get(resource); } } - public void setNamer(Class resource, Namer namer) { + public void setNamer(Class resource, Namer namer) { map.put(resource, namer); } } diff --git a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/cache/Keys.groovy b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/cache/Keys.groovy index 0fd61b7f466..c4cfff1b6fd 100644 --- a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/cache/Keys.groovy +++ b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/cache/Keys.groovy @@ -25,6 +25,8 @@ import com.netflix.spinnaker.moniker.Moniker import groovy.util.logging.Slf4j import org.springframework.stereotype.Component +import javax.annotation.Nullable + @Slf4j @Component("GoogleKeys") class Keys implements KeyParser { @@ -78,6 +80,7 @@ class Keys implements KeyParser { return false } + @Nullable static Map parse(String key) { def parts = key.split(':') diff --git a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/model/GoogleServerGroup.groovy b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/model/GoogleServerGroup.groovy index ca7ad4df4ac..f5826a74c0e 100644 --- a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/model/GoogleServerGroup.groovy +++ b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/model/GoogleServerGroup.groovy @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo import com.google.api.services.compute.model.AutoscalingPolicy import com.google.api.services.compute.model.InstanceGroupManagerActionsSummary import com.google.api.services.compute.model.InstanceGroupManagerAutoHealingPolicy +import com.google.api.services.compute.model.ServiceAccount import com.google.api.services.compute.model.StatefulPolicy -import com.google.api.services.iam.v1.model.ServiceAccount import com.netflix.spinnaker.clouddriver.google.GoogleCloudProvider import com.netflix.spinnaker.clouddriver.google.model.loadbalancing.GoogleHttpLoadBalancingPolicy import com.netflix.spinnaker.clouddriver.google.model.loadbalancing.GoogleLoadBalancerView diff --git a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleRegionalServerGroupCachingAgent.groovy b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleRegionalServerGroupCachingAgent.groovy index 7ebf47e08f3..c5add35334e 100644 --- a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleRegionalServerGroupCachingAgent.groovy +++ b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleRegionalServerGroupCachingAgent.groovy @@ -21,9 +21,11 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.google.api.client.googleapis.batch.json.JsonBatchCallback import com.google.api.client.googleapis.json.GoogleJsonError import com.google.api.client.http.HttpHeaders +import com.google.api.services.compute.Compute import com.google.api.services.compute.ComputeRequest import com.google.api.services.compute.model.* import com.netflix.frigga.Names +import com.netflix.frigga.ami.AppVersion import com.netflix.spectator.api.Registry import com.netflix.spinnaker.cats.agent.AgentDataType import com.netflix.spinnaker.cats.agent.CacheResult @@ -33,6 +35,7 @@ import com.netflix.spinnaker.cats.provider.ProviderCache import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent import com.netflix.spinnaker.clouddriver.cache.OnDemandMetricsSupport import com.netflix.spinnaker.clouddriver.google.GoogleCloudProvider +import com.netflix.spinnaker.clouddriver.google.GoogleExecutor import com.netflix.spinnaker.clouddriver.google.GoogleExecutorTraits import com.netflix.spinnaker.clouddriver.google.batch.GoogleBatchRequest import com.netflix.spinnaker.clouddriver.google.cache.CacheResultBuilder @@ -42,7 +45,9 @@ import com.netflix.spinnaker.clouddriver.google.model.GoogleDistributionPolicy import com.netflix.spinnaker.clouddriver.google.model.GoogleInstance import com.netflix.spinnaker.clouddriver.google.model.GoogleServerGroup import com.netflix.spinnaker.clouddriver.google.model.callbacks.Utils +import com.netflix.spinnaker.clouddriver.google.model.loadbalancing.GoogleHttpLoadBalancingPolicy import com.netflix.spinnaker.clouddriver.google.provider.agent.util.PaginatedRequest +import com.netflix.spinnaker.clouddriver.google.security.AccountForClient import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials import com.netflix.spinnaker.moniker.Moniker import groovy.transform.Canonical @@ -53,6 +58,7 @@ import java.util.concurrent.TimeUnit import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.INFORMATIVE import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.* +import static com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil.* @Slf4j class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent implements OnDemandAgent, GoogleExecutorTraits { @@ -133,7 +139,7 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i GoogleBatchRequest instanceGroupsRequest = buildGoogleBatchRequest() GoogleBatchRequest autoscalerRequest = buildGoogleBatchRequest() - List instanceTemplates = GoogleZonalServerGroupCachingAgent.fetchInstanceTemplates(cachingAgent, compute, project) + List instanceTemplates = fetchInstanceTemplates(cachingAgent, compute, project) List instances = GCEUtil.fetchInstances(this, credentials) InstanceGroupManagerCallbacks instanceGroupManagerCallbacks = new InstanceGroupManagerCallbacks( @@ -170,6 +176,26 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i serverGroups } + static List fetchInstanceTemplates(AbstractGoogleCachingAgent cachingAgent, Compute compute, String project) { + List instanceTemplates = new PaginatedRequest(cachingAgent) { + @Override + protected ComputeRequest request (String pageToken) { + return compute.instanceTemplates().list(project).setPageToken(pageToken) + } + + @Override + String getNextPageToken(InstanceTemplateList t) { + return t.getNextPageToken(); + } + }.timeExecute( + { InstanceTemplateList list -> list.getItems() }, + "compute.instanceTemplates.list", GoogleExecutor.TAG_SCOPE, GoogleExecutor.SCOPE_GLOBAL, + "account", AccountForClient.getAccount(compute) + ) + + return instanceTemplates + } + @Override boolean handles(OnDemandAgent.OnDemandType type, String cloudProvider) { type == OnDemandAgent.OnDemandType.ServerGroup && cloudProvider == GoogleCloudProvider.ID @@ -286,7 +312,7 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i } log.debug("Writing cache entry for cluster key ${clusterKey} adding relationships for application ${appKey} and server group ${serverGroupKey}") - GoogleZonalServerGroupCachingAgent.populateLoadBalancerKeys(serverGroup, loadBalancerKeys, accountName, region) + populateLoadBalancerKeys(serverGroup, loadBalancerKeys, accountName, region) loadBalancerKeys.each { String loadBalancerKey -> cacheResultBuilder.namespace(LOAD_BALANCERS.ns).keep(loadBalancerKey).with { @@ -294,7 +320,7 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i } } - if (GoogleZonalServerGroupCachingAgent.shouldUseOnDemandData(cacheResultBuilder, serverGroupKey)) { + if (shouldUseOnDemandData(cacheResultBuilder, serverGroupKey)) { moveOnDemandDataToNamespace(cacheResultBuilder, serverGroup) } else { cacheResultBuilder.namespace(SERVER_GROUPS.ns).keep(serverGroupKey).with { @@ -317,6 +343,11 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i cacheResultBuilder.build() } + static boolean shouldUseOnDemandData(CacheResultBuilder cacheResultBuilder, String serverGroupKey) { + CacheData cacheData = cacheResultBuilder.onDemand.toKeep[serverGroupKey] + return cacheData ? cacheData.attributes.cacheTime >= cacheResultBuilder.startTime : false + } + void moveOnDemandDataToNamespace(CacheResultBuilder cacheResultBuilder, GoogleServerGroup googleServerGroup) { def serverGroupKey = getServerGroupKey(googleServerGroup) @@ -455,7 +486,7 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i Utils.deriveNetworkLoadBalancerNamesFromTargetPoolUrls(instanceGroupManager.getTargetPools()) InstanceTemplate template = instanceTemplates.find { it -> it.getName() == instanceTemplateName } - GoogleZonalServerGroupCachingAgent.populateServerGroupWithTemplate(serverGroup, providerCache, loadBalancerNames, + populateServerGroupWithTemplate(serverGroup, providerCache, loadBalancerNames, template, accountName, project, objectMapper) def instanceMetadata = template?.properties?.metadata if (instanceMetadata) { @@ -465,6 +496,156 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i } } + static void populateServerGroupWithTemplate(GoogleServerGroup serverGroup, ProviderCache providerCache, + List loadBalancerNames, InstanceTemplate instanceTemplate, + String accountName, String project, ObjectMapper objectMapper) { + serverGroup.with { + networkName = Utils.decorateXpnResourceIdIfNeeded(project, instanceTemplate?.properties?.networkInterfaces?.getAt(0)?.network) + canIpForward = instanceTemplate?.properties?.canIpForward + instanceTemplateTags = instanceTemplate?.properties?.tags?.items + instanceTemplateServiceAccounts = instanceTemplate?.properties?.serviceAccounts + instanceTemplateLabels = instanceTemplate?.properties?.labels + launchConfig.with { + launchConfigurationName = instanceTemplate?.name + instanceType = instanceTemplate?.properties?.machineType + minCpuPlatform = instanceTemplate?.properties?.minCpuPlatform + } + } + // "instanceTemplate = instanceTemplate" in the above ".with{ }" blocks doesn't work because Groovy thinks it's + // assigning the same variable to itself, instead of to the "launchConfig" entry + serverGroup.launchConfig.instanceTemplate = instanceTemplate + + sortWithBootDiskFirst(serverGroup) + + def sourceImageUrl = instanceTemplate?.properties?.disks?.find { disk -> + disk.boot + }?.initializeParams?.sourceImage + if (sourceImageUrl) { + serverGroup.launchConfig.imageId = Utils.getLocalName(sourceImageUrl) + + def imageKey = Keys.getImageKey(accountName, serverGroup.launchConfig.imageId) + def image = providerCache.get(IMAGES.ns, imageKey) + + extractBuildInfo(image?.attributes?.image?.description, serverGroup) + } + + def instanceMetadata = instanceTemplate?.properties?.metadata + setLoadBalancerMetadataOnInstance(loadBalancerNames, instanceMetadata, serverGroup, objectMapper) + } + + static void populateLoadBalancerKeys(GoogleServerGroup serverGroup, List loadBalancerKeys, String accountName, String region) { + serverGroup.asg.get(REGIONAL_LOAD_BALANCER_NAMES).each { String loadBalancerName -> + loadBalancerKeys << Keys.getLoadBalancerKey(region, accountName, loadBalancerName) + } + serverGroup.asg.get(GLOBAL_LOAD_BALANCER_NAMES).each { String loadBalancerName -> + loadBalancerKeys << Keys.getLoadBalancerKey("global", accountName, loadBalancerName) + } + } + + static void sortWithBootDiskFirst(GoogleServerGroup serverGroup) { + // Ensure that the boot disk is listed as the first persistent disk. + if (serverGroup.launchConfig.instanceTemplate?.properties?.disks) { + def persistentDisks = serverGroup.launchConfig.instanceTemplate.properties.disks.findAll { it.type == "PERSISTENT" } + + if (persistentDisks && !persistentDisks.first().boot) { + def sortedDisks = [] + def firstBootDisk = persistentDisks.find { it.boot } + + if (firstBootDisk) { + sortedDisks << firstBootDisk + } + + sortedDisks.addAll(serverGroup.launchConfig.instanceTemplate.properties.disks.findAll { !it.boot }) + serverGroup.launchConfig.instanceTemplate.properties.disks = sortedDisks + } + } + } + + /** + * Set load balancing metadata on the server group from the instance template. + * + * @param loadBalancerNames -- Network load balancer names specified by target pools. + * @param instanceMetadata -- Metadata associated with the instance template. + * @param serverGroup -- Server groups built from the instance template. + */ + static void setLoadBalancerMetadataOnInstance(List loadBalancerNames, + Metadata instanceMetadata, + GoogleServerGroup serverGroup, + ObjectMapper objectMapper) { + if (instanceMetadata) { + def metadataMap = Utils.buildMapFromMetadata(instanceMetadata) + def regionalLBNameList = metadataMap?.get(REGIONAL_LOAD_BALANCER_NAMES)?.split(",") + def globalLBNameList = metadataMap?.get(GLOBAL_LOAD_BALANCER_NAMES)?.split(",") + def backendServiceList = metadataMap?.get(BACKEND_SERVICE_NAMES)?.split(",") + def policyJson = metadataMap?.get(LOAD_BALANCING_POLICY) + + if (globalLBNameList) { + serverGroup.asg.put(GLOBAL_LOAD_BALANCER_NAMES, globalLBNameList) + } + if (backendServiceList) { + serverGroup.asg.put(BACKEND_SERVICE_NAMES, backendServiceList) + } + if (policyJson) { + serverGroup.asg.put(LOAD_BALANCING_POLICY, objectMapper.readValue(policyJson, GoogleHttpLoadBalancingPolicy)) + } + + if (regionalLBNameList) { + serverGroup.asg.put(REGIONAL_LOAD_BALANCER_NAMES, regionalLBNameList) + + // The isDisabled property of a server group is set based on whether there are associated target pools, + // and whether the metadata of the server group contains a list of load balancers to actually associate + // the server group with. + // We set the disabled state for L4 lBs here (before writing into the cache) and calculate + // the L7 disabled state when we read the server groups from the cache. + serverGroup.setDisabled(loadBalancerNames.empty) + } + } + } + + static void extractBuildInfo(String imageDescription, GoogleServerGroup googleServerGroup) { + if (imageDescription) { + def descriptionTokens = imageDescription?.tokenize(",") + def appVersionTag = findTagValue(descriptionTokens, "appversion") + Map buildInfo = null + + if (appVersionTag) { + def appVersion = AppVersion.parseName(appVersionTag) + + if (appVersion) { + buildInfo = [package_name: appVersion.packageName, version: appVersion.version, commit: appVersion.commit] as Map + + if (appVersion.buildJobName) { + buildInfo.jenkins = [name: appVersion.buildJobName, number: appVersion.buildNumber] + } + + def buildHostTag = findTagValue(descriptionTokens, "build_host") + + if (buildHostTag && buildInfo.containsKey("jenkins")) { + ((Map)buildInfo.jenkins).host = buildHostTag + } + + def buildInfoUrlTag = findTagValue(descriptionTokens, "build_info_url") + + if (buildInfoUrlTag) { + buildInfo.buildInfoUrl = buildInfoUrlTag + } + } + + if (buildInfo) { + googleServerGroup.buildInfo = buildInfo + } + } + } + } + + static String findTagValue(List descriptionTokens, String tagKey) { + def matchingKeyValuePair = descriptionTokens?.find { keyValuePair -> + keyValuePair.trim().startsWith("$tagKey: ") + } + + matchingKeyValuePair ? matchingKeyValuePair.trim().substring(tagKey.length() + 2) : null + } + class AutoscalerSingletonCallback extends JsonBatchCallback { GoogleServerGroup serverGroup diff --git a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.groovy b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.groovy deleted file mode 100644 index 8df32483d99..00000000000 --- a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.groovy +++ /dev/null @@ -1,723 +0,0 @@ -/* - * Copyright 2016 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.clouddriver.google.provider.agent - -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import com.google.api.client.googleapis.batch.json.JsonBatchCallback -import com.google.api.client.googleapis.json.GoogleJsonError -import com.google.api.client.http.HttpHeaders -import com.google.api.services.compute.Compute -import com.google.api.services.compute.ComputeRequest -import com.google.api.services.compute.model.* -import com.netflix.frigga.Names -import com.netflix.frigga.ami.AppVersion -import com.netflix.spectator.api.Registry -import com.netflix.spinnaker.cats.agent.AgentDataType -import com.netflix.spinnaker.cats.agent.CacheResult -import com.netflix.spinnaker.cats.cache.CacheData -import com.netflix.spinnaker.cats.cache.DefaultCacheData -import com.netflix.spinnaker.cats.provider.ProviderCache -import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent -import com.netflix.spinnaker.clouddriver.cache.OnDemandMetricsSupport -import com.netflix.spinnaker.clouddriver.google.GoogleCloudProvider -import com.netflix.spinnaker.clouddriver.google.GoogleExecutor -import com.netflix.spinnaker.clouddriver.google.GoogleExecutorTraits -import com.netflix.spinnaker.clouddriver.google.batch.GoogleBatchRequest -import com.netflix.spinnaker.clouddriver.google.cache.CacheResultBuilder -import com.netflix.spinnaker.clouddriver.google.cache.Keys -import com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil -import com.netflix.spinnaker.clouddriver.google.model.GoogleInstance -import com.netflix.spinnaker.clouddriver.google.model.GoogleServerGroup -import com.netflix.spinnaker.clouddriver.google.model.callbacks.Utils -import com.netflix.spinnaker.clouddriver.google.model.loadbalancing.GoogleHttpLoadBalancingPolicy -import com.netflix.spinnaker.clouddriver.google.provider.agent.util.PaginatedRequest -import com.netflix.spinnaker.clouddriver.google.security.AccountForClient -import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials -import com.netflix.spinnaker.moniker.Moniker -import groovy.transform.Canonical -import groovy.util.logging.Slf4j - -import java.util.concurrent.TimeUnit - -import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE -import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.INFORMATIVE -import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.* -import static com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil.* - -@Slf4j -class GoogleZonalServerGroupCachingAgent extends AbstractGoogleCachingAgent implements OnDemandAgent, GoogleExecutorTraits { - - final String region - final long maxMIGPageSize - - final Set providedDataTypes = [ - AUTHORITATIVE.forType(SERVER_GROUPS.ns), - AUTHORITATIVE.forType(APPLICATIONS.ns), - INFORMATIVE.forType(CLUSTERS.ns), - INFORMATIVE.forType(LOAD_BALANCERS.ns), - ] as Set - - String agentType = "${accountName}/${region}/${GoogleZonalServerGroupCachingAgent.simpleName}" - String onDemandAgentType = "${agentType}-OnDemand" - final OnDemandMetricsSupport metricsSupport - - GoogleZonalServerGroupCachingAgent(String clouddriverUserAgentApplicationName, - GoogleNamedAccountCredentials credentials, - ObjectMapper objectMapper, - Registry registry, - String region, - long maxMIGPageSize) { - super(clouddriverUserAgentApplicationName, - credentials, - objectMapper, - registry) - this.region = region - this.maxMIGPageSize = maxMIGPageSize - this.metricsSupport = new OnDemandMetricsSupport( - registry, - this, - "${GoogleCloudProvider.ID}:${OnDemandAgent.OnDemandType.ServerGroup}") - } - - @Override - CacheResult loadData(ProviderCache providerCache) { - def cacheResultBuilder = new CacheResultBuilder(startTime: System.currentTimeMillis()) - - List serverGroups = getServerGroups(providerCache) - def serverGroupKeys = serverGroups.collect { getServerGroupKey(it) } - - providerCache.getAll(ON_DEMAND.ns, serverGroupKeys).each { CacheData cacheData -> - // Ensure that we don't overwrite data that was inserted by the `handle` method while we retrieved the - // managed instance groups. Furthermore, cache data that hasn't been moved to the proper namespace needs to be - // updated in the ON_DEMAND cache, so don't evict data without a processedCount > 0. - if (cacheData.attributes.cacheTime < cacheResultBuilder.startTime && cacheData.attributes.processedCount > 0) { - cacheResultBuilder.onDemand.toEvict << cacheData.id - } else { - cacheResultBuilder.onDemand.toKeep[cacheData.id] = cacheData - } - } - - CacheResult cacheResults = buildCacheResult(cacheResultBuilder, serverGroups) - - cacheResults.cacheResults[ON_DEMAND.ns].each { CacheData cacheData -> - cacheData.attributes.processedTime = System.currentTimeMillis() - cacheData.attributes.processedCount = (cacheData.attributes.processedCount ?: 0) + 1 - } - - cacheResults - } - - private List getServerGroups(ProviderCache providerCache) { - constructServerGroups(providerCache) - } - - private GoogleServerGroup getServerGroup(ProviderCache providerCache, String onDemandServerGroupName) { - def serverGroups = constructServerGroups(providerCache, onDemandServerGroupName) - serverGroups ? serverGroups.first() : null - } - - private List constructServerGroups(ProviderCache providerCache, String onDemandServerGroupName = null) { - GoogleZonalServerGroupCachingAgent cachingAgent = this - List zones = credentials.getZonesFromRegion(region) - List serverGroups = [] - - GoogleBatchRequest igmRequest = buildGoogleBatchRequest() - GoogleBatchRequest instanceGroupsRequest = buildGoogleBatchRequest() - GoogleBatchRequest autoscalerRequest = buildGoogleBatchRequest() - - List instanceTemplates = fetchInstanceTemplates(cachingAgent, compute, project) - List instances = GCEUtil.fetchInstances(this, credentials) - - zones?.each { String zone -> - InstanceGroupManagerCallbacks instanceGroupManagerCallbacks = new InstanceGroupManagerCallbacks( - providerCache: providerCache, - serverGroups: serverGroups, - zone: zone, - instanceGroupsRequest: instanceGroupsRequest, - autoscalerRequest: autoscalerRequest, - instances: instances) - if (onDemandServerGroupName) { - InstanceGroupManagerCallbacks.InstanceGroupManagerSingletonCallback igmCallback = - instanceGroupManagerCallbacks.newInstanceGroupManagerSingletonCallback(instanceTemplates, instances) - igmRequest.queue(compute.instanceGroupManagers().get(project, zone, onDemandServerGroupName), igmCallback) - } else { - InstanceGroupManagerCallbacks.InstanceGroupManagerListCallback igmlCallback = - instanceGroupManagerCallbacks.newInstanceGroupManagerListCallback(instanceTemplates, instances) - new PaginatedRequest(cachingAgent) { - @Override - ComputeRequest request(String pageToken) { - return compute.instanceGroupManagers().list(project, zone).setMaxResults(maxMIGPageSize).setPageToken(pageToken) - } - - @Override - String getNextPageToken(InstanceGroupManagerList instanceGroupManagerList) { - return instanceGroupManagerList.getNextPageToken() - } - }.queue(igmRequest, igmlCallback, "ZonalServerGroupCaching.igm") - } - } - executeIfRequestsAreQueued(igmRequest, "ZonalServerGroupCaching.igm") - executeIfRequestsAreQueued(instanceGroupsRequest, "ZonalServerGroupCaching.instanceGroups") - executeIfRequestsAreQueued(autoscalerRequest, "ZonalServerGroupCaching.autoscaler") - - serverGroups - } - - static List fetchInstanceTemplates(AbstractGoogleCachingAgent cachingAgent, Compute compute, String project) { - List instanceTemplates = new PaginatedRequest(cachingAgent) { - @Override - protected ComputeRequest request (String pageToken) { - return compute.instanceTemplates().list(project).setPageToken(pageToken) - } - - @Override - String getNextPageToken(InstanceTemplateList t) { - return t.getNextPageToken(); - } - }.timeExecute( - { InstanceTemplateList list -> list.getItems() }, - "compute.instanceTemplates.list", GoogleExecutor.TAG_SCOPE, GoogleExecutor.SCOPE_GLOBAL, - "account", AccountForClient.getAccount(compute) - ) - - return instanceTemplates - } - - @Override - boolean handles(OnDemandType type, String cloudProvider) { - type == OnDemandType.ServerGroup && cloudProvider == GoogleCloudProvider.ID - } - - @Override - OnDemandResult handle(ProviderCache providerCache, Map data) { - if (!data.containsKey("serverGroupName") || data.account != accountName || data.region != region) { - return null - } - - GoogleServerGroup serverGroup = metricsSupport.readData { - getServerGroup(providerCache, data.serverGroupName as String) - } - - if (serverGroup?.regional) { - return null - } - - String serverGroupKey - Collection identifiers = [] - - if (serverGroup) { - serverGroupKey = getServerGroupKey(serverGroup) - } else { - serverGroupKey = Keys.getServerGroupKey(data.serverGroupName as String, null, accountName, region, "*") - - // No server group was found, so need to find identifiers for all zonal server groups in the region. - identifiers = providerCache.filterIdentifiers(SERVER_GROUPS.ns, serverGroupKey) - } - - def cacheResultBuilder = new CacheResultBuilder(startTime: Long.MAX_VALUE) - CacheResult result = metricsSupport.transformData { - buildCacheResult(cacheResultBuilder, serverGroup ? [serverGroup] : []) - } - - if (result.cacheResults.values().flatten().empty) { - // Avoid writing an empty onDemand cache record (instead delete any that may have previously existed). - providerCache.evictDeletedItems(ON_DEMAND.ns, identifiers) - } else { - metricsSupport.onDemandStore { - def cacheData = new DefaultCacheData( - serverGroupKey, - TimeUnit.MINUTES.toSeconds(10) as Integer, // ttl - [ - cacheTime : System.currentTimeMillis(), - cacheResults : objectMapper.writeValueAsString(result.cacheResults), - processedCount: 0, - processedTime : null - ], - [:] - ) - - providerCache.putCacheData(ON_DEMAND.ns, cacheData) - } - } - - Map> evictions = [:].withDefault {_ -> []} - if (!serverGroup) { - evictions[SERVER_GROUPS.ns].addAll(identifiers) - } - - log.debug("On demand cache refresh succeeded. Data: ${data}. Added ${serverGroup ? 1 : 0} items to the cache. Evicted ${evictions[SERVER_GROUPS.ns]}.") - - return new OnDemandResult( - sourceAgentType: getOnDemandAgentType(), - cacheResult: result, - evictions: evictions, - // Do not include "authoritativeTypes" here, as it will result in all other cache entries getting deleted! - ) - } - - @Override - Collection pendingOnDemandRequests(ProviderCache providerCache) { - def keyOwnedByThisAgent = { Map parsedKey -> - parsedKey && parsedKey.account == accountName && parsedKey.region == region && parsedKey.zone - } - - def keys = providerCache.getIdentifiers(ON_DEMAND.ns).findAll { String key -> - keyOwnedByThisAgent(Keys.parse(key)) - } - - providerCache.getAll(ON_DEMAND.ns, keys).collect { CacheData cacheData -> - def details = Keys.parse(cacheData.id) - - [ - details : details, - moniker : cacheData.attributes.moniker, - cacheTime : cacheData.attributes.cacheTime, - processedCount: cacheData.attributes.processedCount, - processedTime : cacheData.attributes.processedTime - ] - } - - } - - private CacheResult buildCacheResult(CacheResultBuilder cacheResultBuilder, List serverGroups) { - log.debug "Describing items in $agentType" - - serverGroups.each { GoogleServerGroup serverGroup -> - Moniker moniker = naming.deriveMoniker(serverGroup) - def applicationName = moniker.app - def clusterName = moniker.cluster - def serverGroupKey = getServerGroupKey(serverGroup) - def clusterKey = Keys.getClusterKey(accountName, applicationName, clusterName) - def appKey = Keys.getApplicationKey(applicationName) - - def loadBalancerKeys = [] - def instanceKeys = serverGroup?.instances?.collect { Keys.getInstanceKey(accountName, region, it.name) } ?: [] - - cacheResultBuilder.namespace(APPLICATIONS.ns).keep(appKey).with { - attributes.name = applicationName - relationships[CLUSTERS.ns].add(clusterKey) - relationships[INSTANCES.ns].addAll(instanceKeys) - } - - cacheResultBuilder.namespace(CLUSTERS.ns).keep(clusterKey).with { - attributes.name = clusterName - attributes.accountName = accountName - attributes.moniker = moniker - relationships[APPLICATIONS.ns].add(appKey) - relationships[SERVER_GROUPS.ns].add(serverGroupKey) - relationships[INSTANCES.ns].addAll(instanceKeys) - } - log.debug("Writing cache entry for cluster key ${clusterKey} adding relationships for application ${appKey} and server group ${serverGroupKey}") - - populateLoadBalancerKeys(serverGroup, loadBalancerKeys, accountName, region) - - loadBalancerKeys.each { String loadBalancerKey -> - cacheResultBuilder.namespace(LOAD_BALANCERS.ns).keep(loadBalancerKey).with { - relationships[SERVER_GROUPS.ns].add(serverGroupKey) - } - } - - if (shouldUseOnDemandData(cacheResultBuilder, serverGroupKey)) { - moveOnDemandDataToNamespace(cacheResultBuilder, serverGroup) - } else { - cacheResultBuilder.namespace(SERVER_GROUPS.ns).keep(serverGroupKey).with { - attributes = objectMapper.convertValue(serverGroup, ATTRIBUTES) - relationships[APPLICATIONS.ns].add(appKey) - relationships[CLUSTERS.ns].add(clusterKey) - relationships[LOAD_BALANCERS.ns].addAll(loadBalancerKeys) - relationships[INSTANCES.ns].addAll(instanceKeys) - } - } - } - - log.debug("Caching ${cacheResultBuilder.namespace(APPLICATIONS.ns).keepSize()} applications in ${agentType}") - log.debug("Caching ${cacheResultBuilder.namespace(CLUSTERS.ns).keepSize()} clusters in ${agentType}") - log.debug("Caching ${cacheResultBuilder.namespace(SERVER_GROUPS.ns).keepSize()} server groups in ${agentType}") - log.debug("Caching ${cacheResultBuilder.namespace(LOAD_BALANCERS.ns).keepSize()} load balancer relationships in ${agentType}") - log.debug("Caching ${cacheResultBuilder.onDemand.toKeep.size()} onDemand entries in ${agentType}") - log.debug("Evicting ${cacheResultBuilder.onDemand.toEvict.size()} onDemand entries in ${agentType}") - - cacheResultBuilder.build() - } - - static boolean shouldUseOnDemandData(CacheResultBuilder cacheResultBuilder, String serverGroupKey) { - CacheData cacheData = cacheResultBuilder.onDemand.toKeep[serverGroupKey] - return cacheData ? cacheData.attributes.cacheTime >= cacheResultBuilder.startTime : false - } - - void moveOnDemandDataToNamespace(CacheResultBuilder cacheResultBuilder, - GoogleServerGroup googleServerGroup) { - def serverGroupKey = getServerGroupKey(googleServerGroup) - Map> onDemandData = objectMapper.readValue( - cacheResultBuilder.onDemand.toKeep[serverGroupKey].attributes.cacheResults as String, - new TypeReference>>() {}) - - onDemandData.each { String namespace, List cacheDatas -> - if (namespace != 'onDemand') { - cacheDatas.each { MutableCacheData cacheData -> - cacheResultBuilder.namespace(namespace).keep(cacheData.id).with { it -> - it.attributes = cacheData.attributes - it.relationships = Utils.mergeOnDemandCacheRelationships(cacheData.relationships, it.relationships) - } - cacheResultBuilder.onDemand.toKeep.remove(cacheData.id) - } - } - } - } - - String getServerGroupKey(GoogleServerGroup googleServerGroup) { - return Keys.getServerGroupKey(googleServerGroup.name, googleServerGroup.view.moniker.cluster, accountName, region, googleServerGroup.zone) - } - - // TODO(lwander) this was taken from the netflix cluster caching, and should probably be shared between all providers. - @Canonical - static class MutableCacheData implements CacheData { - String id - int ttlSeconds = -1 - Map attributes = [:] - Map> relationships = [:].withDefault { [] as Set } - } - - class InstanceGroupManagerCallbacks { - - ProviderCache providerCache - List serverGroups - String zone - GoogleBatchRequest instanceGroupsRequest - GoogleBatchRequest autoscalerRequest - List instances - - InstanceGroupManagerSingletonCallback newInstanceGroupManagerSingletonCallback(List instanceTemplates, List instances) { - return new InstanceGroupManagerSingletonCallback(instanceTemplates: instanceTemplates, instances: instances) - } - - InstanceGroupManagerListCallback newInstanceGroupManagerListCallback(List instanceTemplates, List instances) { - return new InstanceGroupManagerListCallback(instanceTemplates: instanceTemplates, instances: instances) - } - - class InstanceGroupManagerSingletonCallback extends JsonBatchCallback { - - List instanceTemplates - List instances - - @Override - void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException { - // 404 is thrown if the managed instance group does not exist in the given zone. Any other exception needs to be propagated. - if (e.code != 404) { - def errorJson = new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(e) - log.error errorJson - } - } - - @Override - void onSuccess(InstanceGroupManager instanceGroupManager, HttpHeaders responseHeaders) throws IOException { - if (Names.parseName(instanceGroupManager.name)) { - GoogleServerGroup serverGroup = buildServerGroupFromInstanceGroupManager(instanceGroupManager, instances) - serverGroups << serverGroup - - populateInstanceTemplate(providerCache, instanceGroupManager, serverGroup, instanceTemplates) - - def autoscalerCallback = new AutoscalerSingletonCallback(serverGroup: serverGroup) - autoscalerRequest.queue(compute.autoscalers().get(project, zone, serverGroup.name), autoscalerCallback) - } - } - } - - class InstanceGroupManagerListCallback extends JsonBatchCallback implements FailureLogger { - - List instanceTemplates - List instances - - @Override - void onSuccess(InstanceGroupManagerList instanceGroupManagerList, HttpHeaders responseHeaders) throws IOException { - instanceGroupManagerList?.items?.each { InstanceGroupManager instanceGroupManager -> - if (Names.parseName(instanceGroupManager.name)) { - GoogleServerGroup serverGroup = buildServerGroupFromInstanceGroupManager(instanceGroupManager, instances) - serverGroups << serverGroup - - populateInstanceTemplate(providerCache, instanceGroupManager, serverGroup, instanceTemplates) - } - } - - def autoscalerCallback = new AutoscalerAggregatedListCallback(serverGroups: serverGroups) - buildAutoscalerListRequest().queue(autoscalerRequest, autoscalerCallback, - 'GoogleZonalServerGroupCachingAgent.autoscalerAggregatedList') - } - } - - GoogleServerGroup buildServerGroupFromInstanceGroupManager(InstanceGroupManager instanceGroupManager, List instances) { - String zone = Utils.getLocalName(instanceGroupManager.zone) - List groupInstances = instances.findAll { it.getName().startsWith(instanceGroupManager.getBaseInstanceName()) && it.getZone() == zone } - - Map namedPorts = [:] - instanceGroupManager.namedPorts.each { namedPorts[(it.name)] = it.port } - return new GoogleServerGroup( - name: instanceGroupManager.name, - account: accountName, - instances: groupInstances, - region: region, - zone: zone, - namedPorts: namedPorts, - zones: [zone], - selfLink: instanceGroupManager.selfLink, - currentActions: instanceGroupManager.currentActions, - launchConfig: [createdTime: Utils.getTimeFromTimestamp(instanceGroupManager.creationTimestamp)], - asg: [minSize : instanceGroupManager.targetSize, - maxSize : instanceGroupManager.targetSize, - desiredCapacity: instanceGroupManager.targetSize], - statefulPolicy: instanceGroupManager.statefulPolicy, - autoHealingPolicy: instanceGroupManager.autoHealingPolicies?.getAt(0) - ) - } - - void populateInstanceTemplate(ProviderCache providerCache, InstanceGroupManager instanceGroupManager, - GoogleServerGroup serverGroup, List instanceTemplates) { - String instanceTemplateName = Utils.getLocalName(instanceGroupManager.instanceTemplate) - List loadBalancerNames = - Utils.deriveNetworkLoadBalancerNamesFromTargetPoolUrls(instanceGroupManager.getTargetPools()) - InstanceTemplate template = instanceTemplates.find { it -> it.getName() == instanceTemplateName } - populateServerGroupWithTemplate(serverGroup, providerCache, loadBalancerNames, template, accountName, project, objectMapper) - } - } - - static void populateServerGroupWithTemplate(GoogleServerGroup serverGroup, ProviderCache providerCache, - List loadBalancerNames, InstanceTemplate instanceTemplate, - String accountName, String project, ObjectMapper objectMapper) { - serverGroup.with { - networkName = Utils.decorateXpnResourceIdIfNeeded(project, instanceTemplate?.properties?.networkInterfaces?.getAt(0)?.network) - canIpForward = instanceTemplate?.properties?.canIpForward - instanceTemplateTags = instanceTemplate?.properties?.tags?.items - instanceTemplateServiceAccounts = instanceTemplate?.properties?.serviceAccounts - instanceTemplateLabels = instanceTemplate?.properties?.labels - launchConfig.with { - launchConfigurationName = instanceTemplate?.name - instanceType = instanceTemplate?.properties?.machineType - minCpuPlatform = instanceTemplate?.properties?.minCpuPlatform - } - } - // "instanceTemplate = instanceTemplate" in the above ".with{ }" blocks doesn't work because Groovy thinks it's - // assigning the same variable to itself, instead of to the "launchConfig" entry - serverGroup.launchConfig.instanceTemplate = instanceTemplate - - sortWithBootDiskFirst(serverGroup) - - def sourceImageUrl = instanceTemplate?.properties?.disks?.find { disk -> - disk.boot - }?.initializeParams?.sourceImage - if (sourceImageUrl) { - serverGroup.launchConfig.imageId = Utils.getLocalName(sourceImageUrl) - - def imageKey = Keys.getImageKey(accountName, serverGroup.launchConfig.imageId) - def image = providerCache.get(IMAGES.ns, imageKey) - - extractBuildInfo(image?.attributes?.image?.description, serverGroup) - } - - def instanceMetadata = instanceTemplate?.properties?.metadata - setLoadBalancerMetadataOnInstance(loadBalancerNames, instanceMetadata, serverGroup, objectMapper) - } - - static void populateLoadBalancerKeys(GoogleServerGroup serverGroup, List loadBalancerKeys, String accountName, String region) { - serverGroup.asg.get(REGIONAL_LOAD_BALANCER_NAMES).each { String loadBalancerName -> - loadBalancerKeys << Keys.getLoadBalancerKey(region, accountName, loadBalancerName) - } - serverGroup.asg.get(GLOBAL_LOAD_BALANCER_NAMES).each { String loadBalancerName -> - loadBalancerKeys << Keys.getLoadBalancerKey("global", accountName, loadBalancerName) - } - } - - static void sortWithBootDiskFirst(GoogleServerGroup serverGroup) { - // Ensure that the boot disk is listed as the first persistent disk. - if (serverGroup.launchConfig.instanceTemplate?.properties?.disks) { - def persistentDisks = serverGroup.launchConfig.instanceTemplate.properties.disks.findAll { it.type == "PERSISTENT" } - - if (persistentDisks && !persistentDisks.first().boot) { - def sortedDisks = [] - def firstBootDisk = persistentDisks.find { it.boot } - - if (firstBootDisk) { - sortedDisks << firstBootDisk - } - - sortedDisks.addAll(serverGroup.launchConfig.instanceTemplate.properties.disks.findAll { !it.boot }) - serverGroup.launchConfig.instanceTemplate.properties.disks = sortedDisks - } - } - } - - /** - * Set load balancing metadata on the server group from the instance template. - * - * @param loadBalancerNames -- Network load balancer names specified by target pools. - * @param instanceMetadata -- Metadata associated with the instance template. - * @param serverGroup -- Server groups built from the instance template. - */ - static void setLoadBalancerMetadataOnInstance(List loadBalancerNames, - Metadata instanceMetadata, - GoogleServerGroup serverGroup, - ObjectMapper objectMapper) { - if (instanceMetadata) { - def metadataMap = Utils.buildMapFromMetadata(instanceMetadata) - def regionalLBNameList = metadataMap?.get(REGIONAL_LOAD_BALANCER_NAMES)?.split(",") - def globalLBNameList = metadataMap?.get(GLOBAL_LOAD_BALANCER_NAMES)?.split(",") - def backendServiceList = metadataMap?.get(BACKEND_SERVICE_NAMES)?.split(",") - def policyJson = metadataMap?.get(LOAD_BALANCING_POLICY) - - if (globalLBNameList) { - serverGroup.asg.put(GLOBAL_LOAD_BALANCER_NAMES, globalLBNameList) - } - if (backendServiceList) { - serverGroup.asg.put(BACKEND_SERVICE_NAMES, backendServiceList) - } - if (policyJson) { - serverGroup.asg.put(LOAD_BALANCING_POLICY, objectMapper.readValue(policyJson, GoogleHttpLoadBalancingPolicy)) - } - - if (regionalLBNameList) { - serverGroup.asg.put(REGIONAL_LOAD_BALANCER_NAMES, regionalLBNameList) - - // The isDisabled property of a server group is set based on whether there are associated target pools, - // and whether the metadata of the server group contains a list of load balancers to actually associate - // the server group with. - // We set the disabled state for L4 lBs here (before writing into the cache) and calculate - // the L7 disabled state when we read the server groups from the cache. - serverGroup.setDisabled(loadBalancerNames.empty) - } - } - } - - static void extractBuildInfo(String imageDescription, GoogleServerGroup googleServerGroup) { - if (imageDescription) { - def descriptionTokens = imageDescription?.tokenize(",") - def appVersionTag = findTagValue(descriptionTokens, "appversion") - Map buildInfo = null - - if (appVersionTag) { - def appVersion = AppVersion.parseName(appVersionTag) - - if (appVersion) { - buildInfo = [package_name: appVersion.packageName, version: appVersion.version, commit: appVersion.commit] as Map - - if (appVersion.buildJobName) { - buildInfo.jenkins = [name: appVersion.buildJobName, number: appVersion.buildNumber] - } - - def buildHostTag = findTagValue(descriptionTokens, "build_host") - - if (buildHostTag && buildInfo.containsKey("jenkins")) { - ((Map)buildInfo.jenkins).host = buildHostTag - } - - def buildInfoUrlTag = findTagValue(descriptionTokens, "build_info_url") - - if (buildInfoUrlTag) { - buildInfo.buildInfoUrl = buildInfoUrlTag - } - } - - if (buildInfo) { - googleServerGroup.buildInfo = buildInfo - } - } - } - } - - static String findTagValue(List descriptionTokens, String tagKey) { - def matchingKeyValuePair = descriptionTokens?.find { keyValuePair -> - keyValuePair.trim().startsWith("$tagKey: ") - } - - matchingKeyValuePair ? matchingKeyValuePair.trim().substring(tagKey.length() + 2) : null - } - - class AutoscalerSingletonCallback extends JsonBatchCallback { - - GoogleServerGroup serverGroup - - @Override - void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException { - // 404 is thrown if the autoscaler does not exist in the given zone. Any other exception needs to be propagated. - if (e.code != 404) { - def errorJson = new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(e) - log.error errorJson - } - } - - @Override - void onSuccess(Autoscaler autoscaler, HttpHeaders responseHeaders) throws IOException { - serverGroup.autoscalingPolicy = autoscaler.getAutoscalingPolicy() - serverGroup.asg.minSize = serverGroup.autoscalingPolicy.minNumReplicas - serverGroup.asg.maxSize = serverGroup.autoscalingPolicy.maxNumReplicas - - List statusDetails = autoscaler.statusDetails - - if (statusDetails) { - serverGroup.autoscalingMessages = statusDetails.collect { it.message } - } - } - } - - class AutoscalerAggregatedListCallback extends JsonBatchCallback implements FailureLogger { - - List serverGroups - - @Override - void onSuccess(AutoscalerAggregatedList autoscalerAggregatedList, HttpHeaders responseHeaders) throws IOException { - - autoscalerAggregatedList?.items?.each { String location, AutoscalersScopedList autoscalersScopedList -> - if (location.startsWith("zones/")) { - def localZoneName = Utils.getLocalName(location) - def region = localZoneName.substring(0, localZoneName.lastIndexOf('-')) - - autoscalersScopedList.autoscalers.each { Autoscaler autoscaler -> - def migName = Utils.getLocalName(autoscaler.target as String) - def serverGroup = serverGroups.find { - it.name == migName && it.region == region - } - - if (serverGroup) { - serverGroup.autoscalingPolicy = autoscaler.getAutoscalingPolicy() - serverGroup.asg.minSize = serverGroup.autoscalingPolicy.minNumReplicas - serverGroup.asg.maxSize = serverGroup.autoscalingPolicy.maxNumReplicas - - List statusDetails = autoscaler.statusDetails - - if (statusDetails) { - serverGroup.autoscalingMessages = statusDetails.collect { it.message } - } - } - } - } - } - } - } - - PaginatedRequest buildAutoscalerListRequest() { - return new PaginatedRequest(this) { - @Override - protected String getNextPageToken(AutoscalerAggregatedList autoscalerAggregatedList) { - return autoscalerAggregatedList.getNextPageToken() - } - - @Override - protected ComputeRequest request(String pageToken) { - return compute.autoscalers().aggregatedList(project).setPageToken(pageToken) - } - } - } -} diff --git a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.java b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.java new file mode 100644 index 00000000000..f942212ad81 --- /dev/null +++ b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.java @@ -0,0 +1,924 @@ +/* + * Copyright 2019 Google, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.clouddriver.google.provider.agent; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE; +import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.INFORMATIVE; +import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.IMAGES; +import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.APPLICATIONS; +import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.CLUSTERS; +import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.INSTANCES; +import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.LOAD_BALANCERS; +import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.ON_DEMAND; +import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.SERVER_GROUPS; +import static com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil.BACKEND_SERVICE_NAMES; +import static com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil.GLOBAL_LOAD_BALANCER_NAMES; +import static com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil.LOAD_BALANCING_POLICY; +import static com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil.REGIONAL_LOAD_BALANCER_NAMES; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.services.compute.Compute; +import com.google.api.services.compute.Compute.InstanceGroupManagers; +import com.google.api.services.compute.Compute.InstanceGroupManagers.Get; +import com.google.api.services.compute.model.AttachedDisk; +import com.google.api.services.compute.model.Autoscaler; +import com.google.api.services.compute.model.AutoscalerStatusDetails; +import com.google.api.services.compute.model.AutoscalingPolicy; +import com.google.api.services.compute.model.DistributionPolicy; +import com.google.api.services.compute.model.Instance; +import com.google.api.services.compute.model.InstanceGroupManager; +import com.google.api.services.compute.model.InstanceProperties; +import com.google.api.services.compute.model.InstanceTemplate; +import com.google.api.services.compute.model.Metadata.Items; +import com.google.api.services.compute.model.NamedPort; +import com.google.common.base.Splitter; +import com.google.common.base.Splitter.MapSplitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.netflix.frigga.ami.AppVersion; +import com.netflix.spectator.api.Registry; +import com.netflix.spinnaker.cats.agent.AccountAware; +import com.netflix.spinnaker.cats.agent.AgentDataType; +import com.netflix.spinnaker.cats.agent.CacheResult; +import com.netflix.spinnaker.cats.agent.CachingAgent; +import com.netflix.spinnaker.cats.agent.DefaultCacheResult; +import com.netflix.spinnaker.cats.cache.CacheData; +import com.netflix.spinnaker.cats.cache.DefaultCacheData; +import com.netflix.spinnaker.cats.provider.ProviderCache; +import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent; +import com.netflix.spinnaker.clouddriver.cache.OnDemandMetricsSupport; +import com.netflix.spinnaker.clouddriver.google.GoogleCloudProvider; +import com.netflix.spinnaker.clouddriver.google.cache.CacheResultBuilder; +import com.netflix.spinnaker.clouddriver.google.cache.CacheResultBuilder.CacheDataBuilder; +import com.netflix.spinnaker.clouddriver.google.cache.Keys; +import com.netflix.spinnaker.clouddriver.google.compute.BatchPaginatedComputeRequest; +import com.netflix.spinnaker.clouddriver.google.compute.GetFirstBatchComputeRequest; +import com.netflix.spinnaker.clouddriver.google.compute.GoogleComputeApiFactory; +import com.netflix.spinnaker.clouddriver.google.compute.InstanceTemplates; +import com.netflix.spinnaker.clouddriver.google.compute.Instances; +import com.netflix.spinnaker.clouddriver.google.compute.ZoneAutoscalers; +import com.netflix.spinnaker.clouddriver.google.compute.ZoneInstanceGroupManagers; +import com.netflix.spinnaker.clouddriver.google.model.GoogleDistributionPolicy; +import com.netflix.spinnaker.clouddriver.google.model.GoogleInstance; +import com.netflix.spinnaker.clouddriver.google.model.GoogleInstances; +import com.netflix.spinnaker.clouddriver.google.model.GoogleLabeledResource; +import com.netflix.spinnaker.clouddriver.google.model.GoogleServerGroup; +import com.netflix.spinnaker.clouddriver.google.model.callbacks.Utils; +import com.netflix.spinnaker.clouddriver.google.model.loadbalancing.GoogleHttpLoadBalancingPolicy; +import com.netflix.spinnaker.clouddriver.google.provider.GoogleInfrastructureProvider; +import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials; +import com.netflix.spinnaker.clouddriver.names.NamerRegistry; +import com.netflix.spinnaker.moniker.Moniker; +import com.netflix.spinnaker.moniker.Namer; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import javax.annotation.ParametersAreNonnullByDefault; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@ParametersAreNonnullByDefault +public final class GoogleZonalServerGroupCachingAgent + implements CachingAgent, OnDemandAgent, AccountAware { + + private static final ImmutableSet DATA_TYPES = + ImmutableSet.of( + AUTHORITATIVE.forType(SERVER_GROUPS.getNs()), + AUTHORITATIVE.forType(APPLICATIONS.getNs()), + INFORMATIVE.forType(CLUSTERS.getNs()), + INFORMATIVE.forType(LOAD_BALANCERS.getNs())); + + private static final String ON_DEMAND_TYPE = + String.join(":", GoogleCloudProvider.getID(), OnDemandType.ServerGroup.name()); + + private static final Splitter COMMA = Splitter.on(',').omitEmptyStrings().trimResults(); + private static final MapSplitter IMAGE_DESCRIPTION_SPLITTER = + Splitter.on(',').withKeyValueSeparator(": "); + + private final GoogleNamedAccountCredentials credentials; + private final GoogleComputeApiFactory computeApiFactory; + private final String region; + private final OnDemandMetricsSupport onDemandMetricsSupport; + private final Namer naming; + private final ObjectMapper objectMapper; + + public GoogleZonalServerGroupCachingAgent( + GoogleNamedAccountCredentials credentials, + GoogleComputeApiFactory computeApiFactory, + Registry registry, + String region, + ObjectMapper objectMapper) { + this.credentials = credentials; + this.computeApiFactory = computeApiFactory; + this.region = region; + this.onDemandMetricsSupport = new OnDemandMetricsSupport(registry, this, ON_DEMAND_TYPE); + this.objectMapper = objectMapper; + this.naming = + NamerRegistry.lookup() + .withProvider(GoogleCloudProvider.getID()) + .withAccount(credentials.getName()) + .withResource(GoogleLabeledResource.class); + } + + @Override + public CacheResult loadData(ProviderCache providerCache) { + + try { + CacheResultBuilder cacheResultBuilder = new CacheResultBuilder(); + cacheResultBuilder.setStartTime(System.currentTimeMillis()); + + List serverGroups = getServerGroups(providerCache); + + // If an entry in ON_DEMAND was generated _after_ we started our caching run, add it to the + // cacheResultBuilder, since we may use it in buildCacheResult. + // + // We don't evict things unless they've been processed because Orca, after sending an + // on-demand cache refresh, doesn't consider the request "finished" until it calls + // pendingOnDemandRequests and sees a processedCount of 1. In a saner world, Orca would + // probably just trust that if the key wasn't returned by pendingOnDemandRequests, it must + // have been processed. But we don't live in that world. + Set serverGroupKeys = + serverGroups.stream().map(this::getServerGroupKey).collect(toImmutableSet()); + providerCache + .getAll(ON_DEMAND.getNs(), serverGroupKeys) + .forEach( + cacheData -> { + long cacheTime = (long) cacheData.getAttributes().get("cacheTime"); + if (cacheTime < cacheResultBuilder.getStartTime() + && (int) cacheData.getAttributes().get("processedCount") > 0) { + cacheResultBuilder.getOnDemand().getToEvict().add(cacheData.getId()); + } else { + cacheResultBuilder.getOnDemand().getToKeep().put(cacheData.getId(), cacheData); + } + }); + + CacheResult cacheResult = buildCacheResult(cacheResultBuilder, serverGroups); + + // For all the ON_DEMAND entries that we marked as 'toKeep' earlier, here we mark them as + // processed so that they get evicted in future calls to this method. Why can't we just mark + // them as evicted here, though? Why wait for another run? + cacheResult + .getCacheResults() + .get(ON_DEMAND.getNs()) + .forEach( + cacheData -> { + cacheData.getAttributes().put("processedTime", System.currentTimeMillis()); + int processedCount = (Integer) cacheData.getAttributes().get("processedCount"); + cacheData.getAttributes().put("processedCount", processedCount + 1); + }); + + return cacheResult; + } catch (IOException e) { + // CatsOnDemandCacheUpdater handles this + throw new UncheckedIOException(e); + } + } + + @Override + public boolean handles(OnDemandType type, String cloudProvider) { + return OnDemandType.ServerGroup.equals(type) + && GoogleCloudProvider.getID().equals(cloudProvider); + } + + @Nullable + @Override + public OnDemandResult handle(ProviderCache providerCache, Map data) { + + try { + String serverGroupName = (String) data.get("serverGroupName"); + if (serverGroupName == null + || !getAccountName().equals(data.get("account")) + || !region.equals(data.get("region"))) { + return null; + } + + Optional serverGroup = + getMetricsSupport().readData(() -> getServerGroup(serverGroupName, providerCache)); + + CacheResultBuilder cacheResultBuilder = new CacheResultBuilder(); + + if (serverGroup.isPresent()) { + String serverGroupKey = getServerGroupKey(serverGroup.get()); + CacheResult result = + getMetricsSupport() + .transformData( + () -> + buildCacheResult(cacheResultBuilder, ImmutableList.of(serverGroup.get()))); + String cacheResults = objectMapper.writeValueAsString(result.getCacheResults()); + CacheData cacheData = + getMetricsSupport() + .onDemandStore( + () -> + new DefaultCacheData( + serverGroupKey, + /* ttlSeconds= */ (int) Duration.ofMinutes(10).getSeconds(), + ImmutableMap.of( + "cacheTime", + System.currentTimeMillis(), + "cacheResults", + cacheResults, + "processedCount", + 0), + /* relationships= */ ImmutableMap.of())); + providerCache.putCacheData(ON_DEMAND.getNs(), cacheData); + return new OnDemandResult( + getOnDemandAgentType(), result, /* evictions= */ ImmutableMap.of()); + } else { + // If we didn't find this server group, look for any existing ON_DEMAND entries for it (in + // any zone) and evict them. + String serverGroupKey = + Keys.getServerGroupKey( + serverGroupName, /* cluster= */ null, getAccountName(), region, /* zone= */ "*"); + Collection existingIdentifiers = + providerCache.filterIdentifiers(SERVER_GROUPS.getNs(), serverGroupKey); + providerCache.evictDeletedItems(ON_DEMAND.getNs(), existingIdentifiers); + return new OnDemandResult( + getOnDemandAgentType(), + new DefaultCacheResult(ImmutableMap.of()), + ImmutableMap.of(SERVER_GROUPS.getNs(), ImmutableList.copyOf(existingIdentifiers))); + } + } catch (IOException e) { + // CatsOnDemandCacheUpdater handles this + throw new UncheckedIOException(e); + } + } + + @Override + public Collection pendingOnDemandRequests(ProviderCache providerCache) { + List ownedKeys = + providerCache.getIdentifiers(ON_DEMAND.getNs()).stream() + .filter(this::keyOwnedByThisAgent) + .collect(toImmutableList()); + + return providerCache.getAll(ON_DEMAND.getNs(), ownedKeys).stream() + .map( + cacheData -> { + Map map = new HashMap<>(); + map.put("details", Keys.parse(cacheData.getId())); + map.put("moniker", cacheData.getAttributes().get("moniker")); + map.put("cacheTime", cacheData.getAttributes().get("cacheTime")); + map.put("processedCount", cacheData.getAttributes().get("processedCount")); + map.put("processedTime", cacheData.getAttributes().get("processedTime")); + return map; + }) + .collect(toImmutableList()); + } + + private boolean keyOwnedByThisAgent(String key) { + Map parsedKey = Keys.parse(key); + return parsedKey != null + && getAccountName().equals(parsedKey.get("account")) + && region.equals(parsedKey.get("region")) + && parsedKey.get("zone") != null; + } + + private CacheResult buildCacheResult( + CacheResultBuilder cacheResultBuilder, List serverGroups) { + + try { + for (GoogleServerGroup serverGroup : serverGroups) { + + Moniker moniker = naming.deriveMoniker(serverGroup); + + String applicationKey = Keys.getApplicationKey(moniker.getApp()); + String clusterKey = + Keys.getClusterKey(getAccountName(), moniker.getApp(), moniker.getCluster()); + String serverGroupKey = getServerGroupKey(serverGroup); + Set instanceKeys = + serverGroup.getInstances().stream() + .map(instance -> Keys.getInstanceKey(getAccountName(), region, instance.getName())) + .collect(toImmutableSet()); + + CacheDataBuilder application = + cacheResultBuilder.namespace(APPLICATIONS.getNs()).keep(applicationKey); + application.getAttributes().put("name", moniker.getApp()); + application.getRelationships().get(CLUSTERS.getNs()).add(clusterKey); + application.getRelationships().get(INSTANCES.getNs()).addAll(instanceKeys); + + CacheDataBuilder cluster = cacheResultBuilder.namespace(CLUSTERS.getNs()).keep(clusterKey); + cluster.getAttributes().put("name", moniker.getCluster()); + cluster.getAttributes().put("accountName", getAccountName()); + cluster.getAttributes().put("moniker", moniker); + cluster.getRelationships().get(APPLICATIONS.getNs()).add(applicationKey); + cluster.getRelationships().get(SERVER_GROUPS.getNs()).add(serverGroupKey); + cluster.getRelationships().get(INSTANCES.getNs()).addAll(instanceKeys); + + Set loadBalancerKeys = getLoadBalancerKeys(serverGroup); + loadBalancerKeys.forEach( + key -> + cacheResultBuilder + .namespace(LOAD_BALANCERS.getNs()) + .keep(key) + .getRelationships() + .get(SERVER_GROUPS.getNs()) + .add(serverGroupKey)); + + if (shouldUseOnDemandData(cacheResultBuilder, serverGroupKey)) { + moveOnDemandDataToNamespace(cacheResultBuilder, serverGroup); + } else { + CacheDataBuilder serverGroupCacheData = + cacheResultBuilder.namespace(SERVER_GROUPS.getNs()).keep(serverGroupKey); + serverGroupCacheData.setAttributes( + objectMapper.convertValue(serverGroup, new TypeReference>() {})); + serverGroupCacheData.getRelationships().get(APPLICATIONS.getNs()).add(applicationKey); + serverGroupCacheData.getRelationships().get(CLUSTERS.getNs()).add(clusterKey); + serverGroupCacheData + .getRelationships() + .get(LOAD_BALANCERS.getNs()) + .addAll(loadBalancerKeys); + serverGroupCacheData.getRelationships().get(INSTANCES.getNs()).addAll(instanceKeys); + } + } + } catch (IOException e) { + // CatsOnDemandCacheUpdater handles this + throw new UncheckedIOException(e); + } + + return cacheResultBuilder.build(); + } + + private ImmutableSet getLoadBalancerKeys(GoogleServerGroup serverGroup) { + ImmutableSet.Builder loadBalancerKeys = ImmutableSet.builder(); + nullableStream((Collection) serverGroup.getAsg().get(REGIONAL_LOAD_BALANCER_NAMES)) + .map(name -> Keys.getLoadBalancerKey(region, credentials.getName(), name)) + .forEach(loadBalancerKeys::add); + nullableStream((Collection) serverGroup.getAsg().get(GLOBAL_LOAD_BALANCER_NAMES)) + .map(name -> Keys.getLoadBalancerKey("global", credentials.getName(), name)) + .forEach(loadBalancerKeys::add); + return loadBalancerKeys.build(); + } + + private static Stream nullableStream(@Nullable Collection collection) { + return Optional.ofNullable(collection).orElse(ImmutableList.of()).stream(); + } + + private static boolean shouldUseOnDemandData( + CacheResultBuilder cacheResultBuilder, String serverGroupKey) { + CacheData cacheData = cacheResultBuilder.getOnDemand().getToKeep().get(serverGroupKey); + return cacheData != null + && (long) cacheData.getAttributes().get("cacheTime") > cacheResultBuilder.getStartTime(); + } + + private void moveOnDemandDataToNamespace( + CacheResultBuilder cacheResultBuilder, GoogleServerGroup serverGroup) throws IOException { + + String serverGroupKey = getServerGroupKey(serverGroup); + Map> onDemandData = + objectMapper.readValue( + (String) + cacheResultBuilder + .getOnDemand() + .getToKeep() + .get(serverGroupKey) + .getAttributes() + .get("cacheResults"), + new TypeReference>>() {}); + onDemandData.forEach( + (namespace, cacheDatas) -> { + if (namespace.equals(ON_DEMAND.getNs())) { + return; + } + + cacheDatas.forEach( + cacheData -> { + CacheDataBuilder cacheDataBuilder = + cacheResultBuilder.namespace(namespace).keep(cacheData.getId()); + cacheDataBuilder.setAttributes(cacheData.getAttributes()); + cacheDataBuilder.setRelationships( + Utils.mergeOnDemandCacheRelationships( + cacheData.getRelationships(), cacheDataBuilder.getRelationships())); + cacheResultBuilder.getOnDemand().getToKeep().remove(cacheData.getId()); + }); + }); + } + + private String getServerGroupKey(GoogleServerGroup serverGroup) { + return Keys.getServerGroupKey( + serverGroup.getName(), + naming.deriveMoniker(serverGroup).getCluster(), + getAccountName(), + region, + serverGroup.getZone()); + } + + private List getServerGroups(ProviderCache providerCache) throws IOException { + + Collection zones = + Optional.ofNullable(credentials.getZonesFromRegion(region)).orElse(ImmutableList.of()); + + ZoneInstanceGroupManagers managersApi = + computeApiFactory.createZoneInstanceGroupManagers(credentials); + Instances instancesApi = computeApiFactory.createInstances(credentials); + InstanceTemplates instanceTemplatesApi = computeApiFactory.createInstanceTemplates(credentials); + ZoneAutoscalers autoscalersApi = computeApiFactory.createZoneAutoscalers(credentials); + + BatchPaginatedComputeRequest + zoneManagersRequest = computeApiFactory.createPaginatedBatchRequest(credentials); + BatchPaginatedComputeRequest instancesRequest = + computeApiFactory.createPaginatedBatchRequest(credentials); + BatchPaginatedComputeRequest autoscalersRequest = + computeApiFactory.createPaginatedBatchRequest(credentials); + + zones.forEach( + zone -> { + zoneManagersRequest.queue(managersApi.list(zone)); + instancesRequest.queue(instancesApi.list(zone)); + autoscalersRequest.queue(autoscalersApi.list(zone)); + }); + + Collection managers = + zoneManagersRequest.execute("ZonalServerGroupCaching.igm"); + Collection instances = + instancesRequest.execute("ZonalServerGroupCaching.instance").stream() + .map(instance -> GoogleInstances.createFromComputeInstance(instance, credentials)) + .collect(toImmutableList()); + Collection autoscalers = + autoscalersRequest.execute("ZonalServerGroupCaching.autoscaler"); + Collection instanceTemplates = instanceTemplatesApi.list().execute(); + + return constructServerGroups( + providerCache, managers, instances, instanceTemplates, autoscalers); + } + + private Optional getServerGroup(String name, ProviderCache providerCache) { + + Collection zones = credentials.getZonesFromRegion(region); + if (zones == null) { + return Optional.empty(); + } + + ZoneInstanceGroupManagers managersApi = + computeApiFactory.createZoneInstanceGroupManagers(credentials); + Instances instancesApi = computeApiFactory.createInstances(credentials); + InstanceTemplates instanceTemplatesApi = computeApiFactory.createInstanceTemplates(credentials); + ZoneAutoscalers autoscalersApi = computeApiFactory.createZoneAutoscalers(credentials); + + GetFirstBatchComputeRequest zoneManagersRequest = + GetFirstBatchComputeRequest.create(computeApiFactory.createBatchRequest(credentials)); + + try { + for (String zone : zones) { + zoneManagersRequest.queue(managersApi.get(zone, name)); + } + Optional managerOpt = + zoneManagersRequest.execute("ZonalServerGroupCaching.igm"); + + if (!managerOpt.isPresent()) { + return Optional.empty(); + } + + InstanceGroupManager manager = managerOpt.get(); + checkState( + !isNullOrEmpty(manager.getZone()), + "Managed instance group %s did not have a zone.", + manager.getName()); + String zone = Utils.getLocalName(manager.getZone()); + checkState( + !isNullOrEmpty(manager.getInstanceTemplate()), + "Managed instance group %s in zone %s did not have an instanceTemplate.", + manager.getName(), + zone); + + List instances = + instancesApi.list(zone).execute().stream() + .map(instance -> GoogleInstances.createFromComputeInstance(instance, credentials)) + .collect(toImmutableList()); + + // TODO(plumpy): does the autoscaler really always have the same name as the server group? + // The old code did this, but the compute API doesn't require it. Maybe it's a pattern that + // Spinnaker enforces? + List autoscalers = new ArrayList<>(); + autoscalersApi.get(zone, manager.getName()).executeGet().ifPresent(autoscalers::add); + + List instanceTemplates = new ArrayList<>(); + instanceTemplatesApi + .get(Utils.getLocalName(manager.getInstanceTemplate())) + .executeGet() + .ifPresent(instanceTemplates::add); + + return constructServerGroups( + providerCache, ImmutableList.of(manager), instances, instanceTemplates, autoscalers) + .stream() + .findAny(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Value + private static class TargetAndZone { + String target; + + String zone; + + static TargetAndZone forAutoscaler(Autoscaler autoscaler) { + return new TargetAndZone( + Utils.getLocalName(autoscaler.getTarget()), Utils.getLocalName(autoscaler.getZone())); + } + } + + private List constructServerGroups( + ProviderCache providerCache, + Collection managers, + Collection instances, + Collection instanceTemplates, + Collection autoscalers) { + + Map autoscalerMap = + autoscalers.stream() + .collect(toImmutableMap(TargetAndZone::forAutoscaler, scaler -> scaler)); + Map instanceTemplatesMap = + instanceTemplates.stream().collect(toImmutableMap(InstanceTemplate::getName, i -> i)); + return managers.stream() + .map( + manager -> { + String zone = Utils.getLocalName(manager.getZone()); + ImmutableSet ownedInstances = ImmutableSet.of(); + if (manager.getBaseInstanceName() != null) { + ownedInstances = + instances.stream() + .filter( + instance -> + instance.getName().startsWith(manager.getBaseInstanceName())) + .filter(instance -> zone.equals(instance.getZone())) + .collect(toImmutableSet()); + } + TargetAndZone key = new TargetAndZone(manager.getName(), zone); + Autoscaler autoscaler = autoscalerMap.get(key); + InstanceTemplate instanceTemplate = + instanceTemplatesMap.get(Utils.getLocalName(manager.getInstanceTemplate())); + return createServerGroup( + manager, ownedInstances, instanceTemplate, autoscaler, providerCache); + }) + .collect(toImmutableList()); + } + + private GoogleServerGroup createServerGroup( + InstanceGroupManager manager, + ImmutableSet instances, + @Nullable InstanceTemplate instanceTemplate, + @Nullable Autoscaler autoscaler, + ProviderCache providerCache) { + + GoogleServerGroup serverGroup = new GoogleServerGroup(); + serverGroup.setName(manager.getName()); + setRegionConfig(serverGroup, manager); + serverGroup.setAccount(credentials.getName()); + serverGroup.setInstances(instances); + serverGroup.setNamedPorts(convertNamedPorts(manager)); + serverGroup.setSelfLink(manager.getSelfLink()); + serverGroup.setCurrentActions(manager.getCurrentActions()); + + setLaunchConfig(serverGroup, manager, instanceTemplate, providerCache); + setAutoscalerGroup(serverGroup, manager, instanceTemplate); + if (instanceTemplate != null) { + InstanceProperties properties = instanceTemplate.getProperties(); + if (properties != null) { + serverGroup.setCanIpForward(properties.getCanIpForward()); + if (properties.getServiceAccounts() != null) { + serverGroup.setInstanceTemplateServiceAccounts( + ImmutableSet.copyOf(properties.getServiceAccounts())); + } + if (properties.getTags() != null && properties.getTags().getItems() != null) { + serverGroup.setInstanceTemplateTags(ImmutableSet.copyOf(properties.getTags().getItems())); + } + if (properties.getLabels() != null) { + serverGroup.setInstanceTemplateLabels(ImmutableMap.copyOf(properties.getLabels())); + } + if (properties.getNetworkInterfaces() != null + && !properties.getNetworkInterfaces().isEmpty() + && properties.getNetworkInterfaces().get(0) != null) { + serverGroup.setNetworkName( + Utils.decorateXpnResourceIdIfNeeded( + credentials.getProject(), properties.getNetworkInterfaces().get(0).getNetwork())); + } + } + } + serverGroup.setStatefulPolicy(manager.getStatefulPolicy()); + if (manager.getAutoHealingPolicies() != null && !manager.getAutoHealingPolicies().isEmpty()) { + serverGroup.setAutoHealingPolicy(manager.getAutoHealingPolicies().get(0)); + } + populateAutoscaler(serverGroup, autoscaler); + return serverGroup; + } + + private void setRegionConfig(GoogleServerGroup serverGroup, InstanceGroupManager manager) { + + serverGroup.setRegional(manager.getZone() == null); + + if (serverGroup.getRegional()) { + serverGroup.setRegion(Utils.getLocalName(manager.getRegion())); + ImmutableList zones = getZones(manager.getDistributionPolicy()); + serverGroup.setZones(ImmutableSet.copyOf(zones)); + serverGroup.setDistributionPolicy(new GoogleDistributionPolicy(zones)); + } else { + String zone = Utils.getLocalName(manager.getZone()); + serverGroup.setZone(zone); + serverGroup.setZones(ImmutableSet.of(zone)); + serverGroup.setRegion(credentials.regionFromZone(zone)); + } + } + + private static ImmutableList getZones(@Nullable DistributionPolicy distributionPolicy) { + if (distributionPolicy == null || distributionPolicy.getZones() == null) { + return ImmutableList.of(); + } + return distributionPolicy.getZones().stream() + .map(z -> Utils.getLocalName(z.getZone())) + .collect(toImmutableList()); + } + + @Nullable + private static ImmutableMap convertNamedPorts(InstanceGroupManager manager) { + if (manager.getNamedPorts() == null) { + return null; + } + return manager.getNamedPorts().stream() + .filter(namedPort -> namedPort.getName() != null) + .filter(namedPort -> namedPort.getPort() != null) + .collect(toImmutableMap(NamedPort::getName, NamedPort::getPort)); + } + + private void setLaunchConfig( + GoogleServerGroup serverGroup, + InstanceGroupManager manager, + @Nullable InstanceTemplate instanceTemplate, + ProviderCache providerCache) { + + HashMap launchConfig = new HashMap<>(); + launchConfig.put("createdTime", Utils.getTimeFromTimestamp(manager.getCreationTimestamp())); + + if (instanceTemplate != null) { + launchConfig.put("launchConfigurationName", instanceTemplate.getName()); + launchConfig.put("instanceTemplate", instanceTemplate); + if (instanceTemplate.getProperties() != null) { + List disks = getDisks(instanceTemplate); + instanceTemplate.getProperties().setDisks(disks); + if (instanceTemplate.getProperties().getMachineType() != null) { + launchConfig.put("instanceType", instanceTemplate.getProperties().getMachineType()); + } + if (instanceTemplate.getProperties().getMinCpuPlatform() != null) { + launchConfig.put("minCpuPlatform", instanceTemplate.getProperties().getMinCpuPlatform()); + } + setSourceImage(serverGroup, launchConfig, disks, credentials, providerCache); + } + } + serverGroup.setLaunchConfig(copyToImmutableMapWithoutNullValues(launchConfig)); + } + + private static ImmutableList getDisks(InstanceTemplate template) { + + if (template.getProperties() == null || template.getProperties().getDisks() == null) { + return ImmutableList.of(); + } + List persistentDisks = + template.getProperties().getDisks().stream() + .filter(disk -> "PERSISTENT".equals(disk.getType())) + .collect(toImmutableList()); + + if (persistentDisks.isEmpty() || persistentDisks.get(0).getBoot()) { + return ImmutableList.copyOf(template.getProperties().getDisks()); + } + + ImmutableList.Builder sortedDisks = ImmutableList.builder(); + Optional firstBootDisk = + persistentDisks.stream().filter(AttachedDisk::getBoot).findFirst(); + firstBootDisk.ifPresent(sortedDisks::add); + template.getProperties().getDisks().stream() + .filter(disk -> !disk.getBoot()) + .forEach(sortedDisks::add); + return sortedDisks.build(); + } + + private static void setSourceImage( + GoogleServerGroup serverGroup, + Map launchConfig, + List disks, + GoogleNamedAccountCredentials credentials, + ProviderCache providerCache) { + + if (disks.isEmpty()) { + return; + } + // Disks were sorted so boot disk comes first + AttachedDisk firstDisk = disks.get(0); + if (!firstDisk.getBoot()) { + return; + } + + if (firstDisk.getInitializeParams() != null + && firstDisk.getInitializeParams().getSourceImage() != null) { + String sourceImage = Utils.getLocalName(firstDisk.getInitializeParams().getSourceImage()); + launchConfig.put("imageId", sourceImage); + String imageKey = Keys.getImageKey(credentials.getName(), sourceImage); + CacheData image = providerCache.get(IMAGES.getNs(), imageKey); + if (image != null) { + String description = + (String) ((Map) image.getAttributes().get("image")).get("description"); + ImmutableMap buildInfo = createBuildInfo(description); + if (buildInfo != null) { + serverGroup.setBuildInfo(buildInfo); + } + } + } + } + + @Nullable + private static ImmutableMap createBuildInfo(@Nullable String imageDescription) { + if (imageDescription == null) { + return null; + } + ImmutableMap.Builder buildInfo = ImmutableMap.builder(); + Map tags; + try { + tags = IMAGE_DESCRIPTION_SPLITTER.split(imageDescription); + } catch (IllegalArgumentException e) { + return null; + } + if (!tags.containsKey("appversion")) { + return null; + } + AppVersion appversion = AppVersion.parseName(tags.get("appversion")); + if (appversion == null) { + return null; + } + buildInfo + .put("package_name", appversion.getPackageName()) + .put("version", appversion.getVersion()) + .put("commit", appversion.getCommit()); + if (appversion.getBuildJobName() != null) { + Map jenkinsInfo = new HashMap<>(); + jenkinsInfo.put("name", appversion.getBuildJobName()); + jenkinsInfo.put("number", appversion.getBuildNumber()); + if (tags.containsKey("build_host")) { + jenkinsInfo.put("host", tags.get("build_host")); + } + buildInfo.put("jenkins", jenkinsInfo); + } + if (tags.containsKey("build_info_url")) { + buildInfo.put("buildInfoUrl", tags.get("build_info_url")); + } + return buildInfo.build(); + } + + private void setAutoscalerGroup( + GoogleServerGroup serverGroup, + InstanceGroupManager manager, + @Nullable InstanceTemplate instanceTemplate) { + + Map autoscalerGroup = new HashMap<>(); + + if (manager.getTargetSize() != null) { + autoscalerGroup.put("minSize", manager.getTargetSize()); + autoscalerGroup.put("maxSize", manager.getTargetSize()); + autoscalerGroup.put("desiredCapacity", manager.getTargetSize()); + } + + if (instanceTemplate != null + && instanceTemplate.getProperties() != null + && instanceTemplate.getProperties().getMetadata() != null + && instanceTemplate.getProperties().getMetadata().getItems() != null) { + + ImmutableMap metadata = + instanceTemplate.getProperties().getMetadata().getItems().stream() + .filter(item -> item.getKey() != null) + .filter(item -> item.getValue() != null) + .collect(toImmutableMap(Items::getKey, Items::getValue)); + + if (metadata.containsKey(GLOBAL_LOAD_BALANCER_NAMES)) { + autoscalerGroup.put( + GLOBAL_LOAD_BALANCER_NAMES, + COMMA.splitToList(metadata.get(GLOBAL_LOAD_BALANCER_NAMES))); + } + + if (metadata.containsKey(REGIONAL_LOAD_BALANCER_NAMES)) { + autoscalerGroup.put( + REGIONAL_LOAD_BALANCER_NAMES, + COMMA.splitToList(metadata.get(REGIONAL_LOAD_BALANCER_NAMES))); + List loadBalancerNames = + Utils.deriveNetworkLoadBalancerNamesFromTargetPoolUrls(manager.getTargetPools()); + + // The isDisabled property of a server group is set based on whether there are associated + // target pools, + // and whether the metadata of the server group contains a list of load balancers to + // actually + // associate + // the server group with. + // We set the disabled state for L4 lBs here (before writing into the cache) and calculate + // the L7 disabled state when we read the server groups from the cache. + serverGroup.setDisabled(loadBalancerNames.isEmpty()); + } + + if (metadata.containsKey(BACKEND_SERVICE_NAMES)) { + autoscalerGroup.put( + BACKEND_SERVICE_NAMES, COMMA.splitToList(metadata.get(BACKEND_SERVICE_NAMES))); + } + + if (metadata.containsKey(LOAD_BALANCING_POLICY)) { + try { + autoscalerGroup.put( + LOAD_BALANCING_POLICY, + objectMapper.readValue( + metadata.get(LOAD_BALANCING_POLICY), GoogleHttpLoadBalancingPolicy.class)); + } catch (IOException e) { + log.warn("Error parsing load balancing policy", e); + } + } + } + + serverGroup.setAsg(copyToImmutableMapWithoutNullValues(autoscalerGroup)); + } + + private static void populateAutoscaler( + GoogleServerGroup serverGroup, @Nullable Autoscaler autoscaler) { + + if (autoscaler == null) { + return; + } + + AutoscalingPolicy autoscalingPolicy = autoscaler.getAutoscalingPolicy(); + if (autoscalingPolicy != null) { + serverGroup.setAutoscalingPolicy(autoscalingPolicy); + // is asg possibly null??? + HashMap autoscalingGroup = new HashMap<>(serverGroup.getAsg()); + autoscalingGroup.put("minSize", autoscalingPolicy.getMinNumReplicas()); + autoscalingGroup.put("maxSize", autoscalingPolicy.getMaxNumReplicas()); + serverGroup.setAsg(copyToImmutableMapWithoutNullValues(autoscalingGroup)); + } + if (autoscaler.getStatusDetails() != null) { + serverGroup.setAutoscalingMessages( + autoscaler.getStatusDetails().stream() + .map(AutoscalerStatusDetails::getMessage) + .filter(Objects::nonNull) + .collect(toImmutableList())); + } + } + + private static ImmutableMap copyToImmutableMapWithoutNullValues(Map map) { + return map.entrySet().stream() + .filter(e -> e.getValue() != null) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public String getProviderName() { + return GoogleInfrastructureProvider.class.getName(); + } + + @Override + public Collection getProvidedDataTypes() { + return DATA_TYPES; + } + + @Override + public String getAgentType() { + return String.format("%s/%s/%s", getAccountName(), region, getClass().getSimpleName()); + } + + @Override + public String getOnDemandAgentType() { + return getAgentType() + "-OnDemand"; + } + + @Override + public OnDemandMetricsSupport getMetricsSupport() { + return onDemandMetricsSupport; + } + + @Override + public String getAccountName() { + return credentials.getName(); + } +} diff --git a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/config/GoogleInfrastructureProviderConfig.groovy b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/config/GoogleInfrastructureProviderConfig.groovy index 457668512b9..5bfffe4503f 100644 --- a/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/config/GoogleInfrastructureProviderConfig.groovy +++ b/clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/config/GoogleInfrastructureProviderConfig.groovy @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature import com.netflix.spectator.api.Registry import com.netflix.spinnaker.cats.agent.Agent +import com.netflix.spinnaker.clouddriver.google.compute.GoogleComputeApiFactory import com.netflix.spinnaker.config.GoogleConfiguration import com.netflix.spinnaker.clouddriver.google.config.GoogleConfigurationProperties import com.netflix.spinnaker.clouddriver.google.provider.GoogleInfrastructureProvider @@ -46,7 +47,8 @@ class GoogleInfrastructureProviderConfig { GoogleConfigurationProperties googleConfigurationProperties, AccountCredentialsRepository accountCredentialsRepository, ObjectMapper objectMapper, - Registry registry) { + Registry registry, + GoogleComputeApiFactory computeApiFactory) { def googleInfrastructureProvider = new GoogleInfrastructureProvider(Collections.newSetFromMap(new ConcurrentHashMap())) @@ -55,7 +57,8 @@ class GoogleInfrastructureProviderConfig { googleInfrastructureProvider, accountCredentialsRepository, objectMapper, - registry) + registry, + computeApiFactory) googleInfrastructureProvider } @@ -66,7 +69,8 @@ class GoogleInfrastructureProviderConfig { GoogleInfrastructureProvider googleInfrastructureProvider, AccountCredentialsRepository accountCredentialsRepository, ObjectMapper objectMapper, - Registry registry) { + Registry registry, + GoogleComputeApiFactory computeApiFactory) { def scheduledAccounts = ProviderUtils.getScheduledAccounts(googleInfrastructureProvider) def allAccounts = ProviderUtils.buildThreadSafeSetOfAccounts(accountCredentialsRepository, GoogleNamedAccountCredentials) @@ -160,12 +164,11 @@ class GoogleInfrastructureProviderConfig { registry, region, googleConfigurationProperties.maxMIGPageSize) - newlyAddedAgents << new GoogleZonalServerGroupCachingAgent(clouddriverUserAgentApplicationName, - credentials, - objectMapper, + newlyAddedAgents << new GoogleZonalServerGroupCachingAgent(credentials, + computeApiFactory, registry, region, - googleConfigurationProperties.maxMIGPageSize) + objectMapper) } // If there is an agent scheduler, then this provider has been through the AgentController in the past. diff --git a/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleServerGroupCachingAgentSpec.groovy b/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleServerGroupCachingAgentSpec.groovy deleted file mode 100644 index 4cb70cf86be..00000000000 --- a/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleServerGroupCachingAgentSpec.groovy +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2016 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netflix.spinnaker.clouddriver.google.provider.agent - -import com.netflix.spinnaker.clouddriver.google.model.GoogleServerGroup -import spock.lang.Specification -import spock.lang.Unroll - -class GoogleServerGroupCachingAgentSpec extends Specification { - private static final String BUILD_HOST = "http://some-jenkins-host:8080/" - - def "should not set build info if no image description is found"() { - setup: - GoogleServerGroup googleServerGroup = new GoogleServerGroup() - - when: - GoogleZonalServerGroupCachingAgent.extractBuildInfo(null, googleServerGroup) - - then: - !googleServerGroup.buildInfo - - when: - GoogleZonalServerGroupCachingAgent.extractBuildInfo("", googleServerGroup) - - then: - !googleServerGroup.buildInfo - } - - def "should not set build info if no relevant image description is found"() { - setup: - GoogleServerGroup googleServerGroup = new GoogleServerGroup() - - when: - GoogleZonalServerGroupCachingAgent.extractBuildInfo("Some non-appversion image description...", googleServerGroup) - - then: - !googleServerGroup.buildInfo - - when: - GoogleZonalServerGroupCachingAgent.extractBuildInfo("SomeKey1: SomeValue1, SomeKey2: SomeValue2", googleServerGroup) - - then: - !googleServerGroup.buildInfo - } - - def "should set build host if image description contains appversion and build_host"() { - setup: - GoogleServerGroup googleServerGroup = new GoogleServerGroup() - - when: - GoogleZonalServerGroupCachingAgent.extractBuildInfo( - "appversion: somepackage-1.0.0-586499.h150/WE-WAPP-somepackage/150, build_host: $BUILD_HOST", - googleServerGroup) - - then: - with(googleServerGroup.buildInfo) { - package_name == "somepackage" - version == "1.0.0" - commit == "586499" - jenkins == [ - name: "WE-WAPP-somepackage", - number: "150", - host: BUILD_HOST - ] - } - } - - @Unroll - def "should sort disks so boot disk is first persistent disk"() { - setup: - def launchConfig = [instanceTemplate: [properties: [disks: disks]]] - GoogleServerGroup googleServerGroup = new GoogleServerGroup(launchConfig: launchConfig) - - when: - GoogleZonalServerGroupCachingAgent.sortWithBootDiskFirst(googleServerGroup) - - then: - googleServerGroup.launchConfig.instanceTemplate.properties.disks == sortedWithBootFirst - - where: - disks || sortedWithBootFirst - [[boot: true, type: 'PERSISTENT']] || [[boot: true, type: 'PERSISTENT']] - [[boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2']] || [[boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: true, type: 'PERSISTENT', source: 'disk-url-2']] || [[boot: true, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-1']] - [[boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] || [[boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: true, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] || [[boot: true, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] - - // Mix in a SCRATCH disk. - [[boot: true, type: 'PERSISTENT'], [boot: false, type: 'SCRATCH']] || [[boot: true, type: 'PERSISTENT'], [boot: false, type: 'SCRATCH']] - [[boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH']] || [[boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: true, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH']] || [[boot: true, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'SCRATCH']] - [[boot: false, type: 'SCRATCH'], [boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] || [[boot: false, type: 'SCRATCH'], [boot: true, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: true, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] || [[boot: true, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'SCRATCH'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] - - // Boot disk missing (really shouldn't happen, but want to ensure we don't disturb the results). - [[boot: false, type: 'PERSISTENT']] || [[boot: false, type: 'PERSISTENT']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2']] || [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] || [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] - - // Mix in a SCRATCH disk and Boot disk missing. - [[boot: false, type: 'PERSISTENT'], [boot: false, type: 'SCRATCH']] || [[boot: false, type: 'PERSISTENT'], [boot: false, type: 'SCRATCH']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH']] || [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH']] - [[boot: false, type: 'SCRATCH'], [boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] || [[boot: false, type: 'SCRATCH'], [boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] - [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] || [[boot: false, type: 'PERSISTENT', source: 'disk-url-1'], [boot: false, type: 'PERSISTENT', source: 'disk-url-2'], [boot: false, type: 'SCRATCH'], [boot: false, type: 'PERSISTENT', source: 'disk-url-3']] - } - - @Unroll - def "malformed instance properties shouldn't break disk sorting logic"() { - setup: - def launchConfig = [instanceTemplate: instanceTemplate] - GoogleServerGroup googleServerGroup = new GoogleServerGroup(launchConfig: launchConfig) - - when: - GoogleZonalServerGroupCachingAgent.sortWithBootDiskFirst(googleServerGroup) - - then: - googleServerGroup.launchConfig.instanceTemplate == instanceTemplate - - where: - instanceTemplate << [ - null, - [properties: null], - [properties: [:]], - [properties: [disks: null]], - [properties: [disks: []]] - ] - } -} diff --git a/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgentTest.java b/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgentTest.java index d8b97a209b3..3c597d08573 100644 --- a/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgentTest.java +++ b/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgentTest.java @@ -55,6 +55,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.MoreExecutors; import com.netflix.spectator.api.DefaultRegistry; import com.netflix.spinnaker.cats.agent.CacheResult; import com.netflix.spinnaker.cats.cache.CacheData; @@ -64,6 +65,8 @@ import com.netflix.spinnaker.cats.provider.ProviderCache; import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent.OnDemandResult; import com.netflix.spinnaker.clouddriver.google.cache.Keys; +import com.netflix.spinnaker.clouddriver.google.compute.GoogleComputeApiFactory; +import com.netflix.spinnaker.clouddriver.google.deploy.GoogleOperationPoller; import com.netflix.spinnaker.clouddriver.google.model.GoogleInstance; import com.netflix.spinnaker.clouddriver.google.model.GoogleLabeledResource; import com.netflix.spinnaker.clouddriver.google.model.GoogleServerGroup; @@ -77,6 +80,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; import org.assertj.core.data.Offset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -370,6 +374,124 @@ void serverGroupDisksAreSortedProperly() { // Since there is no persistent boot disk, we remove all boot disks. assertThat(diskNames).containsExactly("disk2", "disk4", "disk5"); + + // These are copied from the original test code + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk0")); + assertThat(diskNames).containsExactly("disk0"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1")); + assertThat(diskNames).containsExactly("disk0", "disk1"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk1")); + assertThat(diskNames).containsExactly("disk1", "disk0"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk2")); + assertThat(diskNames).containsExactly("disk0", "disk1", "disk2"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk2")); + assertThat(diskNames).containsExactly("disk1", "disk0", "disk2"); + + // Mix in a SCRATCH disk. + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk1")); + assertThat(diskNames).containsExactly("disk0", "disk1"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk2")); + assertThat(diskNames).containsExactly("disk0", "disk1", "disk2"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk2")); + assertThat(diskNames).containsExactly("disk1", "disk0", "disk2"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk0"), + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk2"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk3")); + assertThat(diskNames).containsExactly("disk0", "disk1", "disk2", "disk3"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(true).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk2"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk3")); + assertThat(diskNames).containsExactly("disk1", "disk0", "disk2", "disk3"); + + // Boot disk missing (really shouldn't happen, but want to ensure we don't disturb the results). + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0")); + assertThat(diskNames).containsExactly("disk0"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1")); + assertThat(diskNames).containsExactly("disk0", "disk1"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk2")); + assertThat(diskNames).containsExactly("disk0", "disk1", "disk2"); + + // Mix in a SCRATCH disk and Boot disk missing. + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk1")); + assertThat(diskNames).containsExactly("disk0", "disk1"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk2")); + assertThat(diskNames).containsExactly("disk0", "disk1", "disk2"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk2"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk3")); + assertThat(diskNames).containsExactly("disk0", "disk1", "disk2", "disk3"); + + diskNames = + retrieveCachedDiskNames( + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk0"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk1"), + new AttachedDisk().setBoot(false).setType("SCRATCH").setDeviceName("disk2"), + new AttachedDisk().setBoot(false).setType("PERSISTENT").setDeviceName("disk3")); + assertThat(diskNames).containsExactly("disk0", "disk1", "disk2", "disk3"); } private List retrieveCachedDiskNames(AttachedDisk... inputDisks) { @@ -851,7 +973,7 @@ void pendingOnDemandRequests_attributes() { assertThat(pendingRequests).hasSize(1); assertThat(getOnlyElement(pendingRequests)) - .containsExactly( + .containsOnly( entry("details", Keys.parse(key)), entry("moniker", moniker("mig1-v001")), entry("cacheTime", 12345), @@ -917,7 +1039,10 @@ void handle_serverGroupDoesNotExistButIsInCache() { void handle_serverGroupExists() throws IOException { Compute compute = new StubComputeFactory() - .setInstanceGroupManagers(instanceGroupManager("myservergroup-v001")) + .setInstanceGroupManagers( + instanceGroupManager("myservergroup-v001") + .setInstanceTemplate( + "http://compute/global/instanceTemplates/my-instance-template")) .create(); GoogleZonalServerGroupCachingAgent cachingAgent = createCachingAgent(compute); ProviderCache providerCache = inMemoryProviderCache(); @@ -982,17 +1107,20 @@ private GoogleServerGroup getOnlyServerGroup(CacheResult cacheResult) { public static GoogleZonalServerGroupCachingAgent createCachingAgent(Compute compute) { return new GoogleZonalServerGroupCachingAgent( - "user-agent", new GoogleNamedAccountCredentials.Builder() .project(PROJECT) .name(ACCOUNT_NAME) .compute(compute) .regionToZonesMap(ImmutableMap.of(REGION, ImmutableList.of(ZONE))) .build(), - new ObjectMapper(), + new GoogleComputeApiFactory( + new GoogleOperationPoller(), + new DefaultRegistry(), + "user-agent", + MoreExecutors.listeningDecorator(Executors.newCachedThreadPool())), new DefaultRegistry(), REGION, - 101L); + new ObjectMapper()); } private static InstanceGroupManager instanceGroupManager(String name) { diff --git a/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/StubComputeFactory.java b/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/StubComputeFactory.java index 0b533ec0546..6f1453ef721 100644 --- a/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/StubComputeFactory.java +++ b/clouddriver-google/src/test/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/StubComputeFactory.java @@ -34,11 +34,13 @@ import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.Autoscaler; import com.google.api.services.compute.model.AutoscalerAggregatedList; +import com.google.api.services.compute.model.AutoscalerList; import com.google.api.services.compute.model.AutoscalersScopedList; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.InstanceAggregatedList; import com.google.api.services.compute.model.InstanceGroupManager; import com.google.api.services.compute.model.InstanceGroupManagerList; +import com.google.api.services.compute.model.InstanceList; import com.google.api.services.compute.model.InstanceTemplate; import com.google.api.services.compute.model.InstanceTemplateList; import com.google.api.services.compute.model.InstancesScopedList; @@ -74,19 +76,29 @@ final class StubComputeFactory { private static final Pattern BATCH_COMPUTE_PATTERN = Pattern.compile("/batch/compute/[-.a-zA-Z0-9]+"); + private static final Pattern GET_ZONAL_IGM_PATTERN = Pattern.compile( COMPUTE_PROJECT_PATH_PREFIX + "/zones/([-a-z0-9]+)/instanceGroupManagers/([-a-zA-Z0-9]+)"); private static final Pattern LIST_ZONAL_IGM_PATTERN = Pattern.compile(COMPUTE_PROJECT_PATH_PREFIX + "/zones/([-a-z0-9]+)/instanceGroupManagers"); - private static final Pattern TEMPLATES_PATTERN = + + private static final Pattern GET_INSTANCE_TEMPLATE_PATTERN = + Pattern.compile(COMPUTE_PROJECT_PATH_PREFIX + "/global/instanceTemplates/([-a-zA-Z0-9]+)"); + private static final Pattern LIST_INSTANCE_TEMPLATES_PATTERN = Pattern.compile(COMPUTE_PROJECT_PATH_PREFIX + "/global/instanceTemplates"); + + private static final Pattern LIST_ZONAL_INSTANCES_PATTERN = + Pattern.compile(COMPUTE_PROJECT_PATH_PREFIX + "/zones/([-a-z0-9]+)/instances"); private static final Pattern AGGREGATED_INSTANCES_PATTERN = Pattern.compile(COMPUTE_PROJECT_PATH_PREFIX + "/aggregated/instances"); - private static final Pattern GET_AUTOSCALER_PATTERN = + + private static final Pattern GET_ZONAL_AUTOSCALER_PATTERN = Pattern.compile( COMPUTE_PROJECT_PATH_PREFIX + "/zones/([-a-z0-9]+)/autoscalers/([-a-zA-Z0-9]+)"); + private static final Pattern LIST_ZONAL_AUTOSCALERS_PATTERN = + Pattern.compile(COMPUTE_PROJECT_PATH_PREFIX + "/zones/([-a-z0-9]+)/autoscalers"); private static final Pattern AGGREGATED_AUTOSCALERS_PATTERN = Pattern.compile(COMPUTE_PROJECT_PATH_PREFIX + "/aggregated/autoscalers"); @@ -123,9 +135,16 @@ Compute create() { .addGetResponse( LIST_ZONAL_IGM_PATTERN, new PathBasedJsonResponseGenerator(this::instanceGroupManagerList)) - .addGetResponse(TEMPLATES_PATTERN, this::instanceTemplateList) + .addGetResponse(GET_INSTANCE_TEMPLATE_PATTERN, this::getInstanceTemplate) + .addGetResponse(LIST_INSTANCE_TEMPLATES_PATTERN, this::instanceTemplateList) + .addGetResponse( + LIST_ZONAL_INSTANCES_PATTERN, + new PathBasedJsonResponseGenerator(this::instanceList)) .addGetResponse(AGGREGATED_INSTANCES_PATTERN, this::instanceAggregatedList) - .addGetResponse(GET_AUTOSCALER_PATTERN, this::getAutoscaler) + .addGetResponse(GET_ZONAL_AUTOSCALER_PATTERN, this::getAutoscaler) + .addGetResponse( + LIST_ZONAL_AUTOSCALERS_PATTERN, + new PathBasedJsonResponseGenerator(this::autoscalerList)) .addGetResponse(AGGREGATED_AUTOSCALERS_PATTERN, this::autoscalerAggregatedList); return new Compute( httpTransport, JacksonFactory.getDefaultInstance(), /* httpRequestInitializer= */ null); @@ -155,10 +174,32 @@ private InstanceGroupManagerList instanceGroupManagerList(String path) { .collect(toImmutableList())); } + private MockLowLevelHttpResponse getInstanceTemplate(MockLowLevelHttpRequest request) { + Matcher matcher = GET_INSTANCE_TEMPLATE_PATTERN.matcher(getPath(request)); + checkState(matcher.matches()); + String name = matcher.group(1); + return instanceTemplates.stream() + .filter(template -> name.equals(template.getName())) + .findFirst() + .map(StubComputeFactory::jsonResponse) + .orElse(errorResponse(404)); + } + private MockLowLevelHttpResponse instanceTemplateList(LowLevelHttpRequest request) { return jsonResponse(new InstanceTemplateList().setItems(instanceTemplates)); } + private InstanceList instanceList(String path) { + Matcher matcher = LIST_ZONAL_INSTANCES_PATTERN.matcher(path); + checkState(matcher.matches()); + String zone = matcher.group(1); + return new InstanceList() + .setItems( + instances.stream() + .filter(instance -> zone.equals(Utils.getLocalName(instance.getZone()))) + .collect(toImmutableList())); + } + private MockLowLevelHttpResponse instanceAggregatedList(LowLevelHttpRequest request) { ImmutableListMultimap instancesMultimap = aggregate(instances, Instance::getZone, /* regionFunction= */ instance -> null); @@ -175,7 +216,7 @@ private MockLowLevelHttpResponse instanceAggregatedList(LowLevelHttpRequest requ } private MockLowLevelHttpResponse getAutoscaler(MockLowLevelHttpRequest request) { - Matcher matcher = GET_AUTOSCALER_PATTERN.matcher(getPath(request)); + Matcher matcher = GET_ZONAL_AUTOSCALER_PATTERN.matcher(getPath(request)); checkState(matcher.matches()); String zone = matcher.group(1); String name = matcher.group(2); @@ -187,6 +228,17 @@ private MockLowLevelHttpResponse getAutoscaler(MockLowLevelHttpRequest request) .orElse(errorResponse(404)); } + private AutoscalerList autoscalerList(String path) { + Matcher matcher = LIST_ZONAL_AUTOSCALERS_PATTERN.matcher(path); + checkState(matcher.matches()); + String zone = matcher.group(1); + return new AutoscalerList() + .setItems( + autoscalers.stream() + .filter(autoscaler -> zone.equals(Utils.getLocalName(autoscaler.getZone()))) + .collect(toImmutableList())); + } + private MockLowLevelHttpResponse autoscalerAggregatedList(LowLevelHttpRequest request) { ImmutableListMultimap autoscalersMultimap = aggregate(autoscalers, Autoscaler::getZone, Autoscaler::getRegion);