feat(provider/oraclebmcs): Network caching agent and provider (#1545)

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/cache/Keys.groovy

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.cache
+
+import com.netflix.spinnaker.clouddriver.oraclebmcs.OracleBMCSCloudProvider
+import groovy.util.logging.Slf4j
+
+@Slf4j
+class Keys {
+
+  static enum Namespace {
+
+    NETWORKS
+
+    static String provider = OracleBMCSCloudProvider.ID
+
+    final String ns
+
+    private Namespace() {
+
+      def parts = name().split('_')
+
+      ns = parts.tail().inject(new StringBuilder(parts.head().toLowerCase())) { val, next -> val.append(next.charAt(0)).append(next.substring(1).toLowerCase()) }
+    }
+
+    String toString() {
+      ns
+    }
+  }
+
+  static Map<String, String> parse(String key) {
+    def parts = key.split(':')
+
+    if (parts.length < 2 || parts[0] != OracleBMCSCloudProvider.ID) {
+      return null
+    }
+
+    def result = [provider: parts[0], type: parts[1]]
+
+    if (result.provider != Namespace.provider) {
+      return null
+    }
+
+    switch (result.type) {
+      case Namespace.NETWORKS.ns:
+        result << [
+          name   : parts[2],
+          id     : parts[3],
+          region : parts[4],
+          account: parts[5]
+        ]
+        break
+      default:
+        return null
+        break
+    }
+
+    result
+  }
+
+  static String getNetworkKey(String networkName,
+                              String networkId,
+                              String region,
+                              String account) {
+    "$OracleBMCSCloudProvider.ID:${Namespace.NETWORKS}:${networkName}:${networkId}:${region}:${account}"
+  }
+
+}

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/model/OracleBMCSNetwork.groovy

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.model
+
+import com.netflix.spinnaker.clouddriver.model.Network
+
+class OracleBMCSNetwork implements Network {
+  String cloudProvider
+  String id
+  String name
+  String account
+  String region
+}

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/OracleBMCSInfrastructureProvider.groovy

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider
+
+import com.netflix.spinnaker.cats.agent.Agent
+import com.netflix.spinnaker.cats.agent.AgentSchedulerAware
+import com.netflix.spinnaker.clouddriver.cache.SearchableProvider
+import com.netflix.spinnaker.clouddriver.oraclebmcs.OracleBMCSCloudProvider
+import com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys
+import com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys.Namespace
+
+class OracleBMCSInfrastructureProvider extends AgentSchedulerAware implements SearchableProvider {
+
+  final Collection<Agent> agents
+
+  final Set<String> defaultCaches = [
+    Namespace.NETWORKS.ns
+  ].asImmutable()
+
+  final Map<String, String> urlMappingTemplates = [:]
+
+  final Map<SearchableProvider.SearchableResource, SearchableProvider.SearchResultHydrator> searchResultHydrators = Collections.emptyMap()
+
+  final String providerName = OracleBMCSCloudProvider.ID
+
+  OracleBMCSInfrastructureProvider(Collection<Agent> agents) {
+    this.agents = agents
+  }
+
+  @Override
+  Map<String, String> parseKey(String key) {
+    return Keys.parse(key)
+  }
+}

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/agent/AbstractOracleBMCSCachingAgent.groovy

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider.agent
+
+import com.fasterxml.jackson.core.type.TypeReference
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.cats.agent.CachingAgent
+import com.netflix.spinnaker.clouddriver.oraclebmcs.OracleBMCSCloudProvider
+import com.netflix.spinnaker.clouddriver.oraclebmcs.security.OracleBMCSNamedAccountCredentials
+
+abstract class AbstractOracleBMCSCachingAgent implements CachingAgent {
+
+  final TypeReference<Map<String, Object>> ATTRIBUTES = new TypeReference<Map<String, Object>>() {}
+  final String clouddriverUserAgentApplicationName // "Spinnaker/${version}" HTTP header string
+  final OracleBMCSNamedAccountCredentials credentials
+  final ObjectMapper objectMapper
+  final String providerName = OracleBMCSCloudProvider.ID
+  final String agentType
+
+
+  AbstractOracleBMCSCachingAgent(ObjectMapper objectMapper, OracleBMCSNamedAccountCredentials credentials, String clouddriverUserAgentApplicationName) {
+    this.objectMapper = objectMapper
+    this.credentials = credentials
+    this.clouddriverUserAgentApplicationName = clouddriverUserAgentApplicationName
+    agentType = "${credentials.name}/${credentials.region}/${this.class.simpleName}"
+  }
+}

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/agent/OracleBMCSNetworkCachingAgent.groovy

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider.agent
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.cats.agent.AgentDataType
+import com.netflix.spinnaker.cats.agent.CacheResult
+import com.netflix.spinnaker.cats.agent.DefaultCacheResult
+import com.netflix.spinnaker.cats.cache.CacheData
+import com.netflix.spinnaker.cats.cache.DefaultCacheData
+import com.netflix.spinnaker.cats.provider.ProviderCache
+import com.netflix.spinnaker.clouddriver.oraclebmcs.cache.Keys
+import com.netflix.spinnaker.clouddriver.oraclebmcs.security.OracleBMCSNamedAccountCredentials
+import com.oracle.bmc.core.model.Vcn
+import com.oracle.bmc.core.requests.ListVcnsRequest
+import groovy.util.logging.Slf4j
+
+@Slf4j
+class OracleBMCSNetworkCachingAgent extends AbstractOracleBMCSCachingAgent {
+
+  final Set<AgentDataType> providedDataTypes = [
+    AgentDataType.Authority.AUTHORITATIVE.forType(Keys.Namespace.NETWORKS.ns)
+  ] as Set
+
+  OracleBMCSNetworkCachingAgent(String clouddriverUserAgentApplicationName,
+                                OracleBMCSNamedAccountCredentials credentials,
+                                ObjectMapper objectMapper) {
+    super(objectMapper, credentials, clouddriverUserAgentApplicationName)
+  }
+
+  @Override
+  CacheResult loadData(ProviderCache providerCache) {
+    List<Vcn> vcns = loadVcns()
+    return buildCacheResult(vcns)
+  }
+
+  List<Vcn> loadVcns() {
+    def response = credentials.networkClient.listVcns(ListVcnsRequest.builder()
+      .compartmentId(credentials.compartmentId)
+      .build())
+    return response.items
+  }
+
+  private CacheResult buildCacheResult(List<Vcn> vcns) {
+    log.info("Describing items in $agentType")
+
+    List<CacheData> data = vcns.collect { Vcn vcn ->
+      if (vcn.lifecycleState != Vcn.LifecycleState.Available) {
+        return null
+      }
+      Map<String, Object> attributes = objectMapper.convertValue(vcn, ATTRIBUTES)
+      new DefaultCacheData(
+        Keys.getNetworkKey(vcn.displayName, vcn.id, credentials.region, credentials.name),
+        attributes,
+        [:]
+      )
+    }
+    data.removeAll {it == null}
+    def cacheData = [(Keys.Namespace.NETWORKS.ns): data]
+    log.info("Caching ${data.size()} items in $agentType")
+    return new DefaultCacheResult(cacheData, [:])
+  }
+
+
+}

clouddriver-oracle-bmcs/src/main/groovy/com/netflix/spinnaker/clouddriver/oraclebmcs/provider/config/OracleBMCSInfrastructureProviderConfig.groovy

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2017 Oracle America, Inc.
+ *
+ * The contents of this file are subject to the Apache License Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * If a copy of the Apache License Version 2.0 was not distributed with this file,
+ * You can obtain one at https://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.netflix.spinnaker.clouddriver.oraclebmcs.provider.config
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SerializationFeature
+import com.netflix.spinnaker.cats.agent.Agent
+import com.netflix.spinnaker.cats.provider.ProviderSynchronizerTypeWrapper
+import com.netflix.spinnaker.clouddriver.oraclebmcs.OracleBMCSConfiguration
+import com.netflix.spinnaker.clouddriver.oraclebmcs.provider.OracleBMCSInfrastructureProvider
+import com.netflix.spinnaker.clouddriver.oraclebmcs.provider.agent.OracleBMCSNetworkCachingAgent
+import com.netflix.spinnaker.clouddriver.oraclebmcs.security.OracleBMCSNamedAccountCredentials
+import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
+import com.netflix.spinnaker.clouddriver.security.ProviderUtils
+import org.springframework.beans.factory.config.ConfigurableBeanFactory
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.context.annotation.*
+
+import java.util.concurrent.ConcurrentHashMap
+
+@Configuration
+@Import(OracleBMCSConfiguration)
+@EnableConfigurationProperties
+class OracleBMCSInfrastructureProviderConfig {
+
+  @Bean
+  @DependsOn('oracleBMCSNamedAccountCredentials')
+  OracleBMCSInfrastructureProvider oracleBMCSInfrastructureProvider(String clouddriverUserAgentApplicationName,
+                                                                    AccountCredentialsRepository accountCredentialsRepository,
+                                                                    ObjectMapper objectMapper) {
+    def oracleBMCSInfrastructureProvider =
+      new OracleBMCSInfrastructureProvider(Collections.newSetFromMap(new ConcurrentHashMap<Agent, Boolean>()))
+
+    synchronizeOracleBMCSInfrastructureProvider(clouddriverUserAgentApplicationName,
+      oracleBMCSInfrastructureProvider,
+      accountCredentialsRepository,
+      objectMapper,
+    )
+
+    return oracleBMCSInfrastructureProvider
+  }
+
+  @Bean
+  OracleBMCSInfrastructureProviderSynchronizerTypeWrapper oracleBMCSInfrastructureProviderSynchronizerTypeWrapper() {
+    new OracleBMCSInfrastructureProviderSynchronizerTypeWrapper()
+  }
+
+  class OracleBMCSInfrastructureProviderSynchronizerTypeWrapper implements ProviderSynchronizerTypeWrapper {
+    @Override
+    Class getSynchronizerType() {
+      return OracleBMCSInfrastructureProviderSynchronizer
+    }
+  }
+
+  class OracleBMCSInfrastructureProviderSynchronizer {}
+
+  @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+  @Bean
+  OracleBMCSInfrastructureProviderSynchronizer synchronizeOracleBMCSInfrastructureProvider(
+    String clouddriverUserAgentApplicationName,
+    OracleBMCSInfrastructureProvider oracleBMCSInfrastructureProvider,
+    AccountCredentialsRepository accountCredentialsRepository,
+    ObjectMapper objectMapper) {
+    def scheduledAccounts = ProviderUtils.getScheduledAccounts(oracleBMCSInfrastructureProvider)
+    def allAccounts = ProviderUtils.buildThreadSafeSetOfAccounts(accountCredentialsRepository,
+      OracleBMCSNamedAccountCredentials)
+
+    objectMapper.enable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
+
+    allAccounts.each { OracleBMCSNamedAccountCredentials credentials ->
+      if (!scheduledAccounts.contains(credentials.name)) {
+        def newlyAddedAgents = []
+
+        newlyAddedAgents << new OracleBMCSNetworkCachingAgent(clouddriverUserAgentApplicationName,
+          credentials,
+          objectMapper)
+
+        // If there is an agent scheduler, then this provider has been through the AgentController in the past.
+        // In that case, we need to do the scheduling here (because accounts have been added to a running system).
+        if (oracleBMCSInfrastructureProvider.agentScheduler) {
+          ProviderUtils.rescheduleAgents(oracleBMCSInfrastructureProvider, newlyAddedAgents)
+        }
+
+        oracleBMCSInfrastructureProvider.agents.addAll(newlyAddedAgents)
+      }
+    }
+
+    return new OracleBMCSInfrastructureProviderSynchronizer()
+  }
+}