Skip to content

Commit

Permalink
fix(provider/cf): added space caching agent (#4544)
Browse files Browse the repository at this point in the history
* fix(pcf/spacecaching): added space caching agent

* refactor(provider/cf): use Guava ImmutableMap.of

Co-authored-by: Zach Smith <zachsmith@Zachs-MacBook-Pro-2.local>
Co-authored-by: Kevin Woo <kevinawoo@gmail.com>
  • Loading branch information
3 people committed Apr 27, 2020
1 parent 86e8029 commit 5b7470a
Show file tree
Hide file tree
Showing 22 changed files with 381 additions and 59 deletions.
Expand Up @@ -54,6 +54,19 @@ public JsonPOJOBuilder.Value findPOJOBuilderConfig(AnnotatedClass ac) {
});
}

public Set<CloudFoundrySpace> findSpacesByAccount(String account) {
return cacheView
.getAll(
SPACES.getNs(),
cacheView.filterIdentifiers(SPACES.getNs(), Keys.getAllSpacesKey(account)))
.stream()
.map(
spaceData ->
objectMapper.convertValue(
spaceData.getAttributes().get("resource"), CloudFoundrySpace.class))
.collect(toSet());
}

public Set<CloudFoundryApplication> findApplicationsByKeys(
Collection<String> keys, Detail detail) {
return cacheView.getAll(APPLICATIONS.getNs(), keys, detail.appFilter()).stream()
Expand Down
Expand Up @@ -78,6 +78,14 @@ public static String getApplicationKey(String app) {
return ID + ":" + Namespace.APPLICATIONS + ":" + app.toLowerCase();
}

public static String getSpaceKey(String account, String region) {
return ID + ":" + Namespace.SPACES + ":" + account + ":" + region;
}

public static String getAllSpacesKey(String account) {
return ID + ":" + Namespace.SPACES + ":" + account + ":*";
}

public static String getAllLoadBalancers() {
return ID + ":" + Namespace.LOAD_BALANCERS + ":*";
}
Expand Down Expand Up @@ -187,7 +195,8 @@ public enum Namespace {
INSTANCES("instances"),
LOAD_BALANCERS("loadBalancers"),
ON_DEMAND("onDemand"),
SERVER_GROUPS("serverGroups");
SERVER_GROUPS("serverGroups"),
SPACES("spaces");

final String ns;

Expand Down
Expand Up @@ -43,7 +43,8 @@ public class CloudFoundryProvider extends AgentSchedulerAware implements Searcha
CLUSTERS.getNs(),
SERVER_GROUPS.getNs(),
INSTANCES.getNs(),
LOAD_BALANCERS.getNs())
LOAD_BALANCERS.getNs(),
SPACES.getNs())
.collect(toSet());

private final Map<SearchableResource, SearchResultHydrator> searchResultHydrators =
Expand Down
Expand Up @@ -33,6 +33,7 @@
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClient;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.Views;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import java.io.IOException;
import java.time.Clock;
import java.util.Collection;
Expand All @@ -50,19 +51,17 @@ abstract class AbstractCloudFoundryCachingAgent
private static final ObjectMapper cacheViewMapper =
new ObjectMapper().disable(MapperFeature.DEFAULT_VIEW_INCLUSION);

private final String account;
private final OnDemandMetricsSupport metricsSupport;
private final CloudFoundryClient client;
private final Clock internalClock;
private final CloudFoundryCredentials credentials;

AbstractCloudFoundryCachingAgent(String account, CloudFoundryClient client, Registry registry) {
this(account, client, registry, Clock.systemDefaultZone());
AbstractCloudFoundryCachingAgent(CloudFoundryCredentials credentials, Registry registry) {
this(credentials, registry, Clock.systemDefaultZone());
}

private AbstractCloudFoundryCachingAgent(
String account, CloudFoundryClient client, Registry registry, Clock internalClock) {
this.account = account;
this.client = client;
CloudFoundryCredentials credentials, Registry registry, Clock internalClock) {
this.credentials = credentials;
cacheViewMapper.setConfig(cacheViewMapper.getSerializationConfig().withView(Views.Cache.class));
this.metricsSupport =
new OnDemandMetricsSupport(
Expand All @@ -72,7 +71,7 @@ private AbstractCloudFoundryCachingAgent(

@Override
public String getAccountName() {
return account;
return credentials.getName();
}

@Override
Expand Down Expand Up @@ -100,6 +99,10 @@ static Map<String, Object> cacheView(Object o) {
"resource", cacheViewMapper.convertValue(o, new TypeReference<Map<String, Object>>() {}));
}

protected CloudFoundryClient getClient() {
return credentials.getClient();
}

Map<String, Collection<ResourceCacheData>> getCacheResultsFromCacheData(CacheData cacheData) {
try {
return cacheViewMapper.readValue(
Expand Down
Expand Up @@ -34,11 +34,11 @@
import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.ResourceCacheData;
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClient;
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.model.RouteId;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.CloudFoundryLoadBalancer;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.CloudFoundrySpace;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import io.vavr.collection.HashMap;
import java.util.*;
import javax.annotation.Nullable;
Expand All @@ -55,8 +55,8 @@ public class CloudFoundryLoadBalancerCachingAgent extends AbstractCloudFoundryCa
Collections.singletonList(AUTHORITATIVE.forType(LOAD_BALANCERS.getNs()));

public CloudFoundryLoadBalancerCachingAgent(
String account, CloudFoundryClient client, Registry registry) {
super(account, client, registry);
CloudFoundryCredentials credentials, Registry registry) {
super(credentials, registry);
}

@Override
Expand Down Expand Up @@ -106,7 +106,7 @@ private CacheData setCacheData(
Map<String, CacheData> onDemandCacheDataToKeep,
CloudFoundryLoadBalancer cloudFoundryLoadBalancer,
long start) {
String account = this.getAccount();
String account = this.getAccountName();
String key = Keys.getLoadBalancerKey(account, cloudFoundryLoadBalancer);
CacheData lbCacheData = onDemandCacheDataToKeep.get(key);
if (lbCacheData != null && (long) lbCacheData.getAttributes().get("cacheTime") > start) {
Expand Down Expand Up @@ -156,7 +156,7 @@ public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
String loadBalancerKey =
Optional.ofNullable(cloudFoundryLoadBalancer)
.map(lb -> Keys.getLoadBalancerKey(account, lb))
.orElse(Keys.getLoadBalancerKey(this.getAccount(), loadBalancerName, region));
.orElse(Keys.getLoadBalancerKey(this.getAccountName(), loadBalancerName, region));
Map<String, Collection<String>> evictions;

DefaultCacheResult loadBalancerCacheResults;
Expand Down
Expand Up @@ -35,9 +35,9 @@
import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.ResourceCacheData;
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClient;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.*;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import com.netflix.spinnaker.moniker.Moniker;
import io.vavr.collection.HashMap;
import java.util.*;
Expand All @@ -59,8 +59,8 @@ public class CloudFoundryServerGroupCachingAgent extends AbstractCloudFoundryCac
AUTHORITATIVE.forType(INSTANCES.getNs()));

public CloudFoundryServerGroupCachingAgent(
String account, CloudFoundryClient client, Registry registry) {
super(account, client, registry);
CloudFoundryCredentials cloudFoundryCredentials, Registry registry) {
super(cloudFoundryCredentials, registry);
}

@Override
Expand Down Expand Up @@ -157,7 +157,7 @@ public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
.getApplications()
.findServerGroupByNameAndSpaceId(serverGroupName, space.getId());

String serverGroupKey = Keys.getServerGroupKey(this.getAccount(), serverGroupName, region);
String serverGroupKey = Keys.getServerGroupKey(this.getAccountName(), serverGroupName, region);
Map<String, Collection<String>> evictions;
DefaultCacheResult serverGroupCacheResults;

Expand Down Expand Up @@ -190,7 +190,7 @@ public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
public Collection<Map> pendingOnDemandRequests(ProviderCache providerCache) {
Collection<String> keys =
providerCache.filterIdentifiers(
ON_DEMAND.getNs(), Keys.getServerGroupKey(this.getAccount(), "*", "*"));
ON_DEMAND.getNs(), Keys.getServerGroupKey(this.getAccountName(), "*", "*"));
return providerCache.getAll(ON_DEMAND.getNs(), keys, RelationshipCacheFilter.none()).stream()
.map(
it -> {
Expand Down Expand Up @@ -243,7 +243,7 @@ private CacheData setServerGroupCacheData(
Map<String, CacheData> onDemandCacheDataToKeep,
CloudFoundryServerGroup serverGroup,
long start) {
String account = this.getAccount();
String account = this.getAccountName();
String key = Keys.getServerGroupKey(account, serverGroup.getName(), serverGroup.getRegion());
CacheData sgCacheData = onDemandCacheDataToKeep.get(key);
if (sgCacheData != null && (long) sgCacheData.getAttributes().get("cacheTime") > start) {
Expand All @@ -270,7 +270,7 @@ private CacheData buildApplicationCacheData(CloudFoundryApplication app) {
}

private CacheData buildClusterCacheData(CloudFoundryCluster cluster) {
String account = this.getAccount();
String account = this.getAccountName();
return new ResourceCacheData(
Keys.getClusterKey(account, cluster.getMoniker().getApp(), cluster.getName()),
cacheView(cluster),
Expand All @@ -282,7 +282,7 @@ private CacheData buildClusterCacheData(CloudFoundryCluster cluster) {
}

private CacheData buildServerGroupCacheData(CloudFoundryServerGroup serverGroup) {
String account = this.getAccount();
String account = this.getAccountName();
return new ResourceCacheData(
Keys.getServerGroupKey(account, serverGroup.getName(), serverGroup.getRegion()),
cacheView(serverGroup),
Expand All @@ -303,7 +303,7 @@ private CacheData buildServerGroupCacheData(CloudFoundryServerGroup serverGroup)

private CacheData buildInstanceCacheData(CloudFoundryInstance instance) {
return new ResourceCacheData(
Keys.getInstanceKey(this.getAccount(), instance.getName()),
Keys.getInstanceKey(this.getAccountName(), instance.getName()),
cacheView(instance),
emptyMap());
}
Expand Down
@@ -0,0 +1,100 @@
/*
* Copyright 2020 Armory, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent;

import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
import static com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys.Namespace.SPACES;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.toSet;

import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.netflix.spectator.api.Registry;
import com.netflix.spinnaker.cats.agent.AgentDataType;
import com.netflix.spinnaker.cats.agent.CacheResult;
import com.netflix.spinnaker.cats.agent.DefaultCacheResult;
import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.cats.provider.ProviderCache;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.ResourceCacheData;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.CloudFoundrySpace;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

@Getter
@Slf4j
public class CloudFoundrySpaceCachingAgent extends AbstractCloudFoundryCachingAgent {

private static final ObjectMapper cacheViewMapper =
new ObjectMapper().disable(MapperFeature.DEFAULT_VIEW_INCLUSION);

private final Collection<AgentDataType> providedDataTypes =
Arrays.asList(AUTHORITATIVE.forType(SPACES.getNs()));

public CloudFoundrySpaceCachingAgent(CloudFoundryCredentials credentials, Registry registry) {
super(credentials, registry);
}

@Override
public CacheResult loadData(ProviderCache providerCache) {
long loadDataStart = this.getInternalClock().millis();
String accountName = getAccountName();
log.info("Caching all spaces in Cloud Foundry account " + accountName);

List<CloudFoundrySpace> spaces = getCredentials().getSpacesLive();

Map<String, Collection<CacheData>> results =
ImmutableMap.of(
SPACES.getNs(),
spaces.stream()
.map(
s ->
new ResourceCacheData(
Keys.getSpaceKey(accountName, s.getRegion()), cacheView(s), emptyMap()))
.collect(toSet()));

log.debug(
"Space cache loaded for Cloud Foundry account {}, ({} sec)",
accountName,
(getInternalClock().millis() - loadDataStart) / 1000);
return new DefaultCacheResult(results, emptyMap());
}

@Override
public boolean handles(OnDemandType type, String cloudProvider) {
return false;
}

@Nullable
@Override
public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
return null;
}

@Override
public Collection<Map> pendingOnDemandRequests(ProviderCache providerCache) {
return null;
}
}
Expand Up @@ -20,6 +20,7 @@
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent.CloudFoundryLoadBalancerCachingAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent.CloudFoundryServerGroupCachingAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent.CloudFoundrySpaceCachingAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository;
import com.netflix.spinnaker.clouddriver.security.ProviderUtils;
Expand Down Expand Up @@ -56,14 +57,13 @@ private void synchronizeCloudFoundryProvider(
if (!scheduledAccounts.contains(credentials.getName())) {
cloudFoundryProvider
.getAgents()
.add(
new CloudFoundryServerGroupCachingAgent(
credentials.getName(), credentials.getClient(), registry));
.add(new CloudFoundryServerGroupCachingAgent(credentials, registry));
cloudFoundryProvider
.getAgents()
.add(
new CloudFoundryLoadBalancerCachingAgent(
credentials.getName(), credentials.getClient(), registry));
.add(new CloudFoundryLoadBalancerCachingAgent(credentials, registry));
cloudFoundryProvider
.getAgents()
.add(new CloudFoundrySpaceCachingAgent(credentials, registry));
}
});
}
Expand Down

0 comments on commit 5b7470a

Please sign in to comment.