feat(provider/oraclebmcs): Subnet caching agent and provider (#1551)

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/cache/Keys.groovy

@@ -16,7 +16,8 @@ class Keys {
 
   static enum Namespace {
 
-    NETWORKS
+    NETWORKS,
+    SUBNETS
 
     static String provider = OracleBMCSCloudProvider.ID
 
@@ -56,6 +57,13 @@ class Keys {
           account: parts[5]
         ]
         break
+      case Namespace.SUBNETS.ns:
+        result << [
+          id     : parts[2],
+          region : parts[3],
+          account: parts[4]
+        ]
+        break
       default:
         return null
         break
@@ -71,4 +79,10 @@ class Keys {
     "$OracleBMCSCloudProvider.ID:${Namespace.NETWORKS}:${networkName}:${networkId}:${region}:${account}"
   }
 
+  static String getSubnetKey(String subnetId,
+                             String region,
+                             String account) {
+    "$OracleBMCSCloudProvider.ID:${Namespace.SUBNETS}:${subnetId}:${region}:${account}"
+  }
+
 }

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/model/OracleBMCSSubnet.groovy

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.model
+
+import com.netflix.spinnaker.clouddriver.model.Subnet
+
+class OracleBMCSSubnet implements Subnet {
+  String type
+  String id
+  String name
+  String account
+  String region
+  String purpose = "n/a"
+  String vcnId
+  String availabilityDomain
+  List<String> securityListIds
+}

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/OracleBMCSInfrastructureProvider.groovy

@@ -20,7 +20,8 @@ class OracleBMCSInfrastructureProvider extends AgentSchedulerAware implements Se
   final Collection<Agent> agents
 
   final Set<String> defaultCaches = [
-    Namespace.NETWORKS.ns
+    Namespace.NETWORKS.ns,
+    Namespace.SUBNETS.ns
   ].asImmutable()
 
   final Map<String, String> urlMappingTemplates = [:]

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/agent/OracleBMCSSubnetCachingAgent.groovy

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider.agent
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.cats.agent.AgentDataType
+import com.netflix.spinnaker.cats.agent.CacheResult
+import com.netflix.spinnaker.cats.agent.DefaultCacheResult
+import com.netflix.spinnaker.cats.cache.CacheData
+import com.netflix.spinnaker.cats.cache.DefaultCacheData
+import com.netflix.spinnaker.cats.provider.ProviderCache
+import com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys
+import com.netflix.spinnaker.clouddriver.oraclebmcs.security.OracleBMCSNamedAccountCredentials
+import com.oracle.bmc.core.model.Subnet
+import com.oracle.bmc.core.model.Vcn
+import com.oracle.bmc.core.requests.ListSubnetsRequest
+import com.oracle.bmc.core.requests.ListVcnsRequest
+import groovy.util.logging.Slf4j
+
+import static com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys.Namespace.SUBNETS
+
+@Slf4j
+class OracleBMCSSubnetCachingAgent extends AbstractOracleBMCSCachingAgent {
+
+  final Set<AgentDataType> providedDataTypes = [
+    AgentDataType.Authority.AUTHORITATIVE.forType(SUBNETS.ns)
+  ] as Set
+
+  OracleBMCSSubnetCachingAgent(String clouddriverUserAgentApplicationName,
+                               OracleBMCSNamedAccountCredentials credentials,
+                               ObjectMapper objectMapper) {
+    super(objectMapper, credentials, clouddriverUserAgentApplicationName)
+  }
+
+  @Override
+  CacheResult loadData(ProviderCache providerCache) {
+    List<Subnet> subnets = loadSubnets()
+    return buildCacheResult(subnets)
+  }
+
+  List<Subnet> loadSubnets() {
+    def listVcnsRequest = ListVcnsRequest.builder().compartmentId(credentials.compartmentId).build()
+    return credentials.networkClient.listVcns(listVcnsRequest).items.collect { Vcn vcn ->
+      def listSubnetsRequest = ListSubnetsRequest.builder().compartmentId(credentials.compartmentId).vcnId(vcn.id).build()
+      credentials.networkClient.listSubnets(listSubnetsRequest).items
+    }.flatten() as List<Subnet>
+  }
+
+  private CacheResult buildCacheResult(List<Subnet> subnets) {
+    log.info("Describing items in ${agentType}")
+
+    List<CacheData> data = subnets.collect { Subnet subnet ->
+      if (subnet.lifecycleState != Subnet.LifecycleState.Available) {
+        return null
+      }
+      Map<String, Object> attributes = objectMapper.convertValue(subnet, ATTRIBUTES)
+      def key = Keys.getSubnetKey(subnet.id, credentials.region, credentials.name)
+      new DefaultCacheData(
+        key,
+        attributes,
+        [:]
+      )
+    }
+    data.removeAll { it == null }
+    def cacheData = [(SUBNETS.ns): data]
+    log.info("Caching ${data.size()} items in ${agentType}")
+    return new DefaultCacheResult(cacheData, [:])
+  }
+}

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/config/OracleBMCSInfrastructureProviderConfig.groovy

@@ -15,6 +15,7 @@ import com.netflix.spinnaker.cats.provider.ProviderSynchronizerTypeWrapper
 import com.netflix.spinnaker.clouddriver.oraclebmcs.OracleBMCSConfiguration
 import com.netflix.spinnaker.clouddriver.oraclebmcs.provider.OracleBMCSInfrastructureProvider
 import com.netflix.spinnaker.clouddriver.oraclebmcs.provider.agent.OracleBMCSNetworkCachingAgent
+import com.netflix.spinnaker.clouddriver.oraclebmcs.provider.agent.OracleBMCSSubnetCachingAgent
 import com.netflix.spinnaker.clouddriver.oraclebmcs.security.OracleBMCSNamedAccountCredentials
 import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
 import com.netflix.spinnaker.clouddriver.security.ProviderUtils
@@ -81,6 +82,10 @@ class OracleBMCSInfrastructureProviderConfig {
           credentials,
           objectMapper)
 
+        newlyAddedAgents << new OracleBMCSSubnetCachingAgent(clouddriverUserAgentApplicationName,
+          credentials,
+          objectMapper)
+
         // If there is an agent scheduler, then this provider has been through the AgentController in the past.
         // In that case, we need to do the scheduling here (because accounts have been added to a running system).
         if (oracleBMCSInfrastructureProvider.agentScheduler) {

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/view/OracleBMCSSubnetProvider.groovy

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider.view
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.cats.cache.Cache
+import com.netflix.spinnaker.cats.cache.CacheData
+import com.netflix.spinnaker.cats.cache.RelationshipCacheFilter
+import com.netflix.spinnaker.clouddriver.model.SubnetProvider
+import com.netflix.spinnaker.clouddriver.oraclebmcs.OracleBMCSCloudProvider
+import com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys
+import com.netflix.spinnaker.clouddriver.oraclebmcs.model.OracleBMCSSubnet
+import com.oracle.bmc.core.model.Subnet
+import groovy.util.logging.Slf4j
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Component
+
+import static com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys.Namespace.SUBNETS
+
+@Slf4j
+@Component
+class OracleBMCSSubnetProvider implements SubnetProvider<OracleBMCSSubnet> {
+
+  final Cache cacheView
+  final ObjectMapper objectMapper
+  final String cloudProvider = OracleBMCSCloudProvider.ID
+
+  @Autowired
+  OracleBMCSSubnetProvider(Cache cacheView, ObjectMapper objectMapper) {
+    this.cacheView = cacheView
+    this.objectMapper = objectMapper
+  }
+
+  @Override
+  Set<OracleBMCSSubnet> getAll() {
+    getAllMatchingKeyPattern(Keys.getSubnetKey('*', '*', '*'))
+  }
+
+  Set<OracleBMCSSubnet> getAllMatchingKeyPattern(String pattern) {
+    loadResults(cacheView.filterIdentifiers(SUBNETS.ns, pattern))
+  }
+
+  Set<OracleBMCSSubnet> loadResults(Collection<String> identifiers) {
+    def data = cacheView.getAll(SUBNETS.ns, identifiers, RelationshipCacheFilter.none())
+    def transformed = data.collect(this.&fromCacheData)
+    return transformed
+  }
+
+  OracleBMCSSubnet fromCacheData(CacheData cacheData) {
+    Subnet subnet = objectMapper.convertValue(cacheData.attributes, Subnet)
+    Map<String, String> parts = Keys.parse(cacheData.id)
+
+    return new OracleBMCSSubnet(
+      type: this.cloudProvider,
+      id: subnet.id,
+      name: subnet.displayName,
+      vcnId: subnet.vcnId,
+      availabilityDomain: subnet.availabilityDomain,
+      securityListIds: subnet.securityListIds,
+      account: parts.account,
+      region: parts.region
+    )
+  }
+}

clouddriver-oracle-bmcs/src/test/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/agent/OracleBMCSSubnetCachingAgentSpec.groovy

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider.agent
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys
+import com.netflix.spinnaker.clouddriver.oraclebmcs.security.OracleBMCSNamedAccountCredentials
+import com.oracle.bmc.Region
+import com.oracle.bmc.core.VirtualNetworkClient
+import com.oracle.bmc.core.model.Subnet
+import com.oracle.bmc.core.model.Vcn
+import com.oracle.bmc.core.responses.ListSubnetsResponse
+import com.oracle.bmc.core.responses.ListVcnsResponse
+import spock.lang.Shared
+import spock.lang.Specification
+
+import static com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys.Namespace.SUBNETS
+
+class OracleBMCSSubnetCachingAgentSpec extends Specification {
+
+  @Shared
+  ObjectMapper objectMapper = new ObjectMapper()
+
+  def "agent has correct agentType"() {
+    setup:
+    def creds = Mock(OracleBMCSNamedAccountCredentials)
+    creds.name = "foo"
+    creds.compartmentId = "bar"
+    creds.region = Region.US_PHOENIX_1.regionId
+    def agent = new OracleBMCSSubnetCachingAgent("", creds, objectMapper)
+    def expectedAgentType = "${creds.name}/${creds.region}/${OracleBMCSSubnetCachingAgent.class.simpleName}"
+
+    when:
+    def agentType = agent.getAgentType()
+
+    then:
+    agentType == expectedAgentType
+  }
+
+  def "agent handles null items in listSubnets rsp"() {
+    setup:
+    def creds = Mock(OracleBMCSNamedAccountCredentials)
+    def networkClient = Mock(VirtualNetworkClient)
+    networkClient.listVcns(_) >> ListVcnsResponse.builder().build()
+    creds.networkClient >> networkClient
+    def agent = new OracleBMCSSubnetCachingAgent("", creds, objectMapper)
+
+    when:
+    def cacheResult = agent.loadData(null)
+
+    then:
+    cacheResult != null
+    cacheResult.cacheResults.containsKey(SUBNETS.ns)
+  }
+
+  def "agent creates correct cache result item, filtering out unavailable subnets"() {
+    setup:
+    def creds = Mock(OracleBMCSNamedAccountCredentials)
+    creds.name = "foo"
+    creds.region = Region.US_PHOENIX_1.regionId
+    def networkClient = Mock(VirtualNetworkClient)
+    def vcnId = "ocid.vcn.123"
+    def vcn = new Vcn(null, null, null, null, null, "My Network", vcnId, Vcn.LifecycleState.Available, null)
+    def subnet = new Subnet("AD1", null, null, null, "My Subnet", "ocid.subnet.123", Subnet.LifecycleState.Available, null, null, null, vcnId, null, null)
+    def subnets = [
+      subnet,
+      new Subnet("AD1", null, null, null, "My Subnet 2", "ocid.subnet.234", Subnet.LifecycleState.Terminated, null, null, null, vcnId, null, null),
+      new Subnet("AD1", null, null, null, "My Subnet 3", "ocid.subnet.567", Subnet.LifecycleState.Provisioning, null, null, null, vcnId, null, null)
+    ]
+
+    networkClient.listVcns(_) >> ListVcnsResponse.builder().items([vcn]).build()
+    networkClient.listSubnets(_) >> ListSubnetsResponse.builder().items(subnets).build()
+    creds.networkClient >> networkClient
+    def agent = new OracleBMCSSubnetCachingAgent("", creds, objectMapper)
+
+    when:
+    def cacheResult = agent.loadData(null)
+
+    then:
+    cacheResult != null
+    cacheResult.cacheResults.containsKey(SUBNETS.ns)
+    cacheResult.cacheResults.get(SUBNETS.ns).size() == 1
+    cacheResult.cacheResults.get(SUBNETS.ns).first().id == Keys.getSubnetKey(subnet.id, creds.region, creds.name)
+    cacheResult.cacheResults.get(SUBNETS.ns).first().attributes.get("id") == subnet.id
+    cacheResult.cacheResults.get(SUBNETS.ns).first().attributes.get("displayName") == subnet.displayName
+  }
+
+
+}

clouddriver-oracle-bmcs/src/test/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/view/OracleBMCSSubnetProviderSpec.groovy

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider.view
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.cats.cache.Cache
+import com.netflix.spinnaker.cats.cache.CacheData
+import com.netflix.spinnaker.clouddriver.oraclebmcs.OracleBMCSCloudProvider
+import com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys
+import spock.lang.Specification
+
+import static com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys.Namespace.SUBNETS
+
+class OracleBMCSSubnetProviderSpec extends Specification {
+
+  def "get all subnets from cache"() {
+    setup:
+    def cache = Mock(Cache)
+    def subnetProvider = new OracleBMCSSubnetProvider(cache, new ObjectMapper())
+    def identifiers = Mock(Collection)
+    def attributes = ["displayName": "My Subnet", "id": "ocid.subnet.123", "availabilityDomain": "AD1", "securityListIds": ["ocid.seclist.123"]]
+    def mockData = Mock(CacheData)
+    Collection<CacheData> cacheData = [mockData]
+    def id = "${OracleBMCSCloudProvider.ID}:${SUBNETS}:ocid.subnet.123:us-phoenix-1:DEFAULT"
+
+    when:
+    def results = subnetProvider.getAll()
+
+    then:
+    1 * cache.filterIdentifiers(SUBNETS.ns, "${OracleBMCSCloudProvider.ID}:$SUBNETS:*:*:*") >> identifiers
+    1 * cache.getAll(SUBNETS.ns, identifiers, _) >> cacheData
+    1 * mockData.attributes >> attributes
+    1 * mockData.id >> id
+    results?.first()?.name == attributes["displayName"]
+    results?.first()?.id == attributes["id"]
+    results?.first()?.region == "us-phoenix-1"
+    results?.first()?.account == "DEFAULT"
+    results?.first()?.availabilityDomain == "AD1"
+    results?.first()?.securityListIds == ["ocid.seclist.123"]
+    noExceptionThrown()
+  }
+}