feat(provider/google): Added caching for external IP addresses. (#1550)

clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/cache/Keys.groovy

@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
 @Slf4j
 class Keys {
   static enum Namespace {
+    ADDRESSES,
     APPLICATIONS,
     BACKEND_SERVICES,
     CLUSTERS,
@@ -66,6 +67,13 @@ class Keys {
     def result = [provider: parts[0], type: parts[1]]
 
     switch (result.type) {
+      case Namespace.ADDRESSES.ns:
+        result << [
+            account: parts[2],
+            region : parts[3],
+            name   : parts[4]
+        ]
+        break
       case Namespace.APPLICATIONS.ns:
         result << [application: parts[2]]
         break
@@ -202,6 +210,12 @@ class Keys {
     "$GoogleCloudProvider.ID:${Namespace.HEALTH_CHECKS}:${account}:${kind}:${healthCheckName}"
   }
 
+  static String getAddressKey(String account,
+                              String region,
+                              String addressName) {
+    "$GoogleCloudProvider.ID:${Namespace.ADDRESSES}:${account}:${region}:${addressName}"
+  }
+
   static String getHttpHealthCheckKey(String account,
                                       String httpHealthCheckName) {
     "$GoogleCloudProvider.ID:${Namespace.HTTP_HEALTH_CHECKS}:${account}:${httpHealthCheckName}"

clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/GoogleInfrastructureProvider.groovy

@@ -34,6 +34,7 @@ class GoogleInfrastructureProvider extends AgentSchedulerAware implements Search
   final String providerName = GoogleInfrastructureProvider.name
 
   final Set<String> defaultCaches = [
+      ADDRESSES.ns,
       APPLICATIONS.ns,
       BACKEND_SERVICES.ns,
       CLUSTERS.ns,
@@ -55,6 +56,7 @@ class GoogleInfrastructureProvider extends AgentSchedulerAware implements Search
   ]
 
   final Map<SearchableResource, SearchableProvider.SearchResultHydrator> searchResultHydrators = [
+    (new GoogleSearchableResource(ADDRESSES.ns)): new AddressResultHydrator(),
     (new GoogleSearchableResource(BACKEND_SERVICES.ns)): new BackendServiceResultHydrator(),
     (new GoogleSearchableResource(HEALTH_CHECKS.ns)): new HealthCheckResultHydrator(),
     (new GoogleSearchableResource(HTTP_HEALTH_CHECKS.ns)): new HttpHealthCheckResultHydrator(),
@@ -66,6 +68,17 @@ class GoogleInfrastructureProvider extends AgentSchedulerAware implements Search
     return Keys.parse(key)
   }
 
+  private static class AddressResultHydrator implements SearchableProvider.SearchResultHydrator {
+
+    @Override
+    Map<String, String> hydrateResult(Cache cacheView, Map<String, String> result, String id) {
+      CacheData addressCacheData  = cacheView.get(ADDRESSES.ns, id)
+      return result + [
+          address: JsonOutput.toJson(addressCacheData.attributes.address)
+      ]
+    }
+  }
+
   private static class BackendServiceResultHydrator implements SearchableProvider.SearchResultHydrator {
 
     @Override

clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleGlobalAddressCachingAgent.groovy

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2017 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.google.provider.agent
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.google.api.services.compute.model.Address
+import com.netflix.spectator.api.Registry
+import com.netflix.spinnaker.cats.agent.AgentDataType
+import com.netflix.spinnaker.cats.agent.CacheResult
+import com.netflix.spinnaker.cats.provider.ProviderCache
+import com.netflix.spinnaker.clouddriver.google.cache.CacheResultBuilder
+import com.netflix.spinnaker.clouddriver.google.cache.Keys
+import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials
+import groovy.util.logging.Slf4j
+
+import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE
+import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.ADDRESSES
+
+@Slf4j
+class GoogleGlobalAddressCachingAgent extends AbstractGoogleCachingAgent {
+
+  final Set<AgentDataType> providedDataTypes = [
+      AUTHORITATIVE.forType(ADDRESSES.ns)
+  ] as Set
+
+  String agentType = "$accountName/$GoogleGlobalAddressCachingAgent.simpleName"
+
+  GoogleGlobalAddressCachingAgent(String clouddriverUserAgentApplicationName,
+                                  GoogleNamedAccountCredentials credentials,
+                                  ObjectMapper objectMapper,
+                                  Registry registry) {
+    super(clouddriverUserAgentApplicationName,
+          credentials,
+          objectMapper,
+          registry)
+  }
+
+  @Override
+  CacheResult loadData(ProviderCache providerCache) {
+    List<Address> addresses = loadAddresses()
+    buildCacheResult(providerCache, addresses)
+  }
+
+  List<Address> loadAddresses() {
+    timeExecute(compute.globalAddresses().list(project),
+                "compute.globalAddresses.list",
+                TAG_SCOPE,
+                SCOPE_GLOBAL).items as List
+  }
+
+  private CacheResult buildCacheResult(ProviderCache _, List<Address> addressList) {
+    log.info("Describing items in ${agentType}")
+
+    def cacheResultBuilder = new CacheResultBuilder()
+
+    addressList.each { Address address ->
+      def addressKey = Keys.getAddressKey(accountName, 'global', address.getName())
+
+      cacheResultBuilder.namespace(ADDRESSES.ns).keep(addressKey).with {
+        attributes.address = address
+      }
+    }
+
+    log.info("Caching ${cacheResultBuilder.namespace(ADDRESSES.ns).keepSize()} items in ${agentType}")
+
+    cacheResultBuilder.build()
+  }
+
+}

clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/agent/GoogleRegionalAddressCachingAgent.groovy

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2017 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.google.provider.agent
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.google.api.services.compute.model.Address
+import com.netflix.spectator.api.Registry
+import com.netflix.spinnaker.cats.agent.AgentDataType
+import com.netflix.spinnaker.cats.agent.CacheResult
+import com.netflix.spinnaker.cats.provider.ProviderCache
+import com.netflix.spinnaker.clouddriver.google.cache.CacheResultBuilder
+import com.netflix.spinnaker.clouddriver.google.cache.Keys
+import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials
+import groovy.util.logging.Slf4j
+
+import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE
+import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.ADDRESSES
+
+@Slf4j
+class GoogleRegionalAddressCachingAgent extends AbstractGoogleCachingAgent {
+
+  final String region
+
+  final Set<AgentDataType> providedDataTypes = [
+      AUTHORITATIVE.forType(ADDRESSES.ns)
+  ] as Set
+
+  String agentType = "$accountName/$region/$GoogleRegionalAddressCachingAgent.simpleName"
+
+  GoogleRegionalAddressCachingAgent(String clouddriverUserAgentApplicationName,
+                                    GoogleNamedAccountCredentials credentials,
+                                    ObjectMapper objectMapper,
+                                    Registry registry,
+                                    String region) {
+    super(clouddriverUserAgentApplicationName,
+          credentials,
+          objectMapper,
+          registry)
+    this.region = region
+  }
+
+  @Override
+  CacheResult loadData(ProviderCache providerCache) {
+    List<Address> addresses = loadAddresses()
+    buildCacheResult(providerCache, addresses)
+  }
+
+  List<Address> loadAddresses() {
+    timeExecute(compute.addresses().list(project, region),
+                "compute.addresses.list",
+                TAG_SCOPE,
+                SCOPE_GLOBAL).items as List
+  }
+
+  private CacheResult buildCacheResult(ProviderCache _, List<Address> addressList) {
+    log.info("Describing items in ${agentType}")
+
+    def cacheResultBuilder = new CacheResultBuilder()
+
+    addressList.each { Address address ->
+      def addressKey = Keys.getAddressKey(accountName, region, address.getName())
+
+      cacheResultBuilder.namespace(ADDRESSES.ns).keep(addressKey).with {
+        attributes.address = address
+      }
+    }
+
+    log.info("Caching ${cacheResultBuilder.namespace(ADDRESSES.ns).keepSize()} items in ${agentType}")
+
+    cacheResultBuilder.build()
+  }
+}

clouddriver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/provider/config/GoogleInfrastructureProviderConfig.groovy

@@ -105,12 +105,22 @@ class GoogleInfrastructureProviderConfig {
                                                           objectMapper,
                                                           registry)
 
+        newlyAddedAgents << new GoogleGlobalAddressCachingAgent(clouddriverUserAgentApplicationName,
+                                                                credentials,
+                                                                objectMapper,
+                                                                registry)
+
         regions.each { String region ->
           newlyAddedAgents << new GoogleSubnetCachingAgent(clouddriverUserAgentApplicationName,
                                                            credentials,
                                                            objectMapper,
                                                            registry,
                                                            region)
+          newlyAddedAgents << new GoogleRegionalAddressCachingAgent(clouddriverUserAgentApplicationName,
+                                                                    credentials,
+                                                                    objectMapper,
+                                                                    registry,
+                                                                    region)
         }
 
         newlyAddedAgents << new GoogleHealthCheckCachingAgent(clouddriverUserAgentApplicationName,