Skip to content

Commit

Permalink
fix(provider/cf): added space caching agent (#4544)
Browse files Browse the repository at this point in the history
* fix(pcf/spacecaching): added space caching agent

* refactor(provider/cf): use Guava ImmutableMap.of

Co-authored-by: Zach Smith <zachsmith@Zachs-MacBook-Pro-2.local>
Co-authored-by: Kevin Woo <kevinawoo@gmail.com>
  • Loading branch information
3 people committed Apr 27, 2020
1 parent 86e8029 commit 5b7470a
Show file tree
Hide file tree
Showing 22 changed files with 381 additions and 59 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,19 @@ public JsonPOJOBuilder.Value findPOJOBuilderConfig(AnnotatedClass ac) {
});
}

public Set<CloudFoundrySpace> findSpacesByAccount(String account) {
return cacheView
.getAll(
SPACES.getNs(),
cacheView.filterIdentifiers(SPACES.getNs(), Keys.getAllSpacesKey(account)))
.stream()
.map(
spaceData ->
objectMapper.convertValue(
spaceData.getAttributes().get("resource"), CloudFoundrySpace.class))
.collect(toSet());
}

public Set<CloudFoundryApplication> findApplicationsByKeys(
Collection<String> keys, Detail detail) {
return cacheView.getAll(APPLICATIONS.getNs(), keys, detail.appFilter()).stream()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ public static String getApplicationKey(String app) {
return ID + ":" + Namespace.APPLICATIONS + ":" + app.toLowerCase();
}

public static String getSpaceKey(String account, String region) {
return ID + ":" + Namespace.SPACES + ":" + account + ":" + region;
}

public static String getAllSpacesKey(String account) {
return ID + ":" + Namespace.SPACES + ":" + account + ":*";
}

public static String getAllLoadBalancers() {
return ID + ":" + Namespace.LOAD_BALANCERS + ":*";
}
Expand Down Expand Up @@ -187,7 +195,8 @@ public enum Namespace {
INSTANCES("instances"),
LOAD_BALANCERS("loadBalancers"),
ON_DEMAND("onDemand"),
SERVER_GROUPS("serverGroups");
SERVER_GROUPS("serverGroups"),
SPACES("spaces");

final String ns;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ public class CloudFoundryProvider extends AgentSchedulerAware implements Searcha
CLUSTERS.getNs(),
SERVER_GROUPS.getNs(),
INSTANCES.getNs(),
LOAD_BALANCERS.getNs())
LOAD_BALANCERS.getNs(),
SPACES.getNs())
.collect(toSet());

private final Map<SearchableResource, SearchResultHydrator> searchResultHydrators =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClient;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.Views;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import java.io.IOException;
import java.time.Clock;
import java.util.Collection;
Expand All @@ -50,19 +51,17 @@ abstract class AbstractCloudFoundryCachingAgent
private static final ObjectMapper cacheViewMapper =
new ObjectMapper().disable(MapperFeature.DEFAULT_VIEW_INCLUSION);

private final String account;
private final OnDemandMetricsSupport metricsSupport;
private final CloudFoundryClient client;
private final Clock internalClock;
private final CloudFoundryCredentials credentials;

AbstractCloudFoundryCachingAgent(String account, CloudFoundryClient client, Registry registry) {
this(account, client, registry, Clock.systemDefaultZone());
AbstractCloudFoundryCachingAgent(CloudFoundryCredentials credentials, Registry registry) {
this(credentials, registry, Clock.systemDefaultZone());
}

private AbstractCloudFoundryCachingAgent(
String account, CloudFoundryClient client, Registry registry, Clock internalClock) {
this.account = account;
this.client = client;
CloudFoundryCredentials credentials, Registry registry, Clock internalClock) {
this.credentials = credentials;
cacheViewMapper.setConfig(cacheViewMapper.getSerializationConfig().withView(Views.Cache.class));
this.metricsSupport =
new OnDemandMetricsSupport(
Expand All @@ -72,7 +71,7 @@ private AbstractCloudFoundryCachingAgent(

@Override
public String getAccountName() {
return account;
return credentials.getName();
}

@Override
Expand Down Expand Up @@ -100,6 +99,10 @@ static Map<String, Object> cacheView(Object o) {
"resource", cacheViewMapper.convertValue(o, new TypeReference<Map<String, Object>>() {}));
}

protected CloudFoundryClient getClient() {
return credentials.getClient();
}

Map<String, Collection<ResourceCacheData>> getCacheResultsFromCacheData(CacheData cacheData) {
try {
return cacheViewMapper.readValue(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@
import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.ResourceCacheData;
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClient;
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.model.RouteId;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.CloudFoundryLoadBalancer;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.CloudFoundrySpace;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import io.vavr.collection.HashMap;
import java.util.*;
import javax.annotation.Nullable;
Expand All @@ -55,8 +55,8 @@ public class CloudFoundryLoadBalancerCachingAgent extends AbstractCloudFoundryCa
Collections.singletonList(AUTHORITATIVE.forType(LOAD_BALANCERS.getNs()));

public CloudFoundryLoadBalancerCachingAgent(
String account, CloudFoundryClient client, Registry registry) {
super(account, client, registry);
CloudFoundryCredentials credentials, Registry registry) {
super(credentials, registry);
}

@Override
Expand Down Expand Up @@ -106,7 +106,7 @@ private CacheData setCacheData(
Map<String, CacheData> onDemandCacheDataToKeep,
CloudFoundryLoadBalancer cloudFoundryLoadBalancer,
long start) {
String account = this.getAccount();
String account = this.getAccountName();
String key = Keys.getLoadBalancerKey(account, cloudFoundryLoadBalancer);
CacheData lbCacheData = onDemandCacheDataToKeep.get(key);
if (lbCacheData != null && (long) lbCacheData.getAttributes().get("cacheTime") > start) {
Expand Down Expand Up @@ -156,7 +156,7 @@ public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
String loadBalancerKey =
Optional.ofNullable(cloudFoundryLoadBalancer)
.map(lb -> Keys.getLoadBalancerKey(account, lb))
.orElse(Keys.getLoadBalancerKey(this.getAccount(), loadBalancerName, region));
.orElse(Keys.getLoadBalancerKey(this.getAccountName(), loadBalancerName, region));
Map<String, Collection<String>> evictions;

DefaultCacheResult loadBalancerCacheResults;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@
import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.ResourceCacheData;
import com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClient;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.*;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import com.netflix.spinnaker.moniker.Moniker;
import io.vavr.collection.HashMap;
import java.util.*;
Expand All @@ -59,8 +59,8 @@ public class CloudFoundryServerGroupCachingAgent extends AbstractCloudFoundryCac
AUTHORITATIVE.forType(INSTANCES.getNs()));

public CloudFoundryServerGroupCachingAgent(
String account, CloudFoundryClient client, Registry registry) {
super(account, client, registry);
CloudFoundryCredentials cloudFoundryCredentials, Registry registry) {
super(cloudFoundryCredentials, registry);
}

@Override
Expand Down Expand Up @@ -157,7 +157,7 @@ public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
.getApplications()
.findServerGroupByNameAndSpaceId(serverGroupName, space.getId());

String serverGroupKey = Keys.getServerGroupKey(this.getAccount(), serverGroupName, region);
String serverGroupKey = Keys.getServerGroupKey(this.getAccountName(), serverGroupName, region);
Map<String, Collection<String>> evictions;
DefaultCacheResult serverGroupCacheResults;

Expand Down Expand Up @@ -190,7 +190,7 @@ public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
public Collection<Map> pendingOnDemandRequests(ProviderCache providerCache) {
Collection<String> keys =
providerCache.filterIdentifiers(
ON_DEMAND.getNs(), Keys.getServerGroupKey(this.getAccount(), "*", "*"));
ON_DEMAND.getNs(), Keys.getServerGroupKey(this.getAccountName(), "*", "*"));
return providerCache.getAll(ON_DEMAND.getNs(), keys, RelationshipCacheFilter.none()).stream()
.map(
it -> {
Expand Down Expand Up @@ -243,7 +243,7 @@ private CacheData setServerGroupCacheData(
Map<String, CacheData> onDemandCacheDataToKeep,
CloudFoundryServerGroup serverGroup,
long start) {
String account = this.getAccount();
String account = this.getAccountName();
String key = Keys.getServerGroupKey(account, serverGroup.getName(), serverGroup.getRegion());
CacheData sgCacheData = onDemandCacheDataToKeep.get(key);
if (sgCacheData != null && (long) sgCacheData.getAttributes().get("cacheTime") > start) {
Expand All @@ -270,7 +270,7 @@ private CacheData buildApplicationCacheData(CloudFoundryApplication app) {
}

private CacheData buildClusterCacheData(CloudFoundryCluster cluster) {
String account = this.getAccount();
String account = this.getAccountName();
return new ResourceCacheData(
Keys.getClusterKey(account, cluster.getMoniker().getApp(), cluster.getName()),
cacheView(cluster),
Expand All @@ -282,7 +282,7 @@ private CacheData buildClusterCacheData(CloudFoundryCluster cluster) {
}

private CacheData buildServerGroupCacheData(CloudFoundryServerGroup serverGroup) {
String account = this.getAccount();
String account = this.getAccountName();
return new ResourceCacheData(
Keys.getServerGroupKey(account, serverGroup.getName(), serverGroup.getRegion()),
cacheView(serverGroup),
Expand All @@ -303,7 +303,7 @@ private CacheData buildServerGroupCacheData(CloudFoundryServerGroup serverGroup)

private CacheData buildInstanceCacheData(CloudFoundryInstance instance) {
return new ResourceCacheData(
Keys.getInstanceKey(this.getAccount(), instance.getName()),
Keys.getInstanceKey(this.getAccountName(), instance.getName()),
cacheView(instance),
emptyMap());
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Copyright 2020 Armory, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent;

import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
import static com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys.Namespace.SPACES;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.toSet;

import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.netflix.spectator.api.Registry;
import com.netflix.spinnaker.cats.agent.AgentDataType;
import com.netflix.spinnaker.cats.agent.CacheResult;
import com.netflix.spinnaker.cats.agent.DefaultCacheResult;
import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.cats.provider.ProviderCache;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.Keys;
import com.netflix.spinnaker.clouddriver.cloudfoundry.cache.ResourceCacheData;
import com.netflix.spinnaker.clouddriver.cloudfoundry.model.CloudFoundrySpace;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

@Getter
@Slf4j
public class CloudFoundrySpaceCachingAgent extends AbstractCloudFoundryCachingAgent {

private static final ObjectMapper cacheViewMapper =
new ObjectMapper().disable(MapperFeature.DEFAULT_VIEW_INCLUSION);

private final Collection<AgentDataType> providedDataTypes =
Arrays.asList(AUTHORITATIVE.forType(SPACES.getNs()));

public CloudFoundrySpaceCachingAgent(CloudFoundryCredentials credentials, Registry registry) {
super(credentials, registry);
}

@Override
public CacheResult loadData(ProviderCache providerCache) {
long loadDataStart = this.getInternalClock().millis();
String accountName = getAccountName();
log.info("Caching all spaces in Cloud Foundry account " + accountName);

List<CloudFoundrySpace> spaces = getCredentials().getSpacesLive();

Map<String, Collection<CacheData>> results =
ImmutableMap.of(
SPACES.getNs(),
spaces.stream()
.map(
s ->
new ResourceCacheData(
Keys.getSpaceKey(accountName, s.getRegion()), cacheView(s), emptyMap()))
.collect(toSet()));

log.debug(
"Space cache loaded for Cloud Foundry account {}, ({} sec)",
accountName,
(getInternalClock().millis() - loadDataStart) / 1000);
return new DefaultCacheResult(results, emptyMap());
}

@Override
public boolean handles(OnDemandType type, String cloudProvider) {
return false;
}

@Nullable
@Override
public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
return null;
}

@Override
public Collection<Map> pendingOnDemandRequests(ProviderCache providerCache) {
return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.CloudFoundryProvider;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent.CloudFoundryLoadBalancerCachingAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent.CloudFoundryServerGroupCachingAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.provider.agent.CloudFoundrySpaceCachingAgent;
import com.netflix.spinnaker.clouddriver.cloudfoundry.security.CloudFoundryCredentials;
import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository;
import com.netflix.spinnaker.clouddriver.security.ProviderUtils;
Expand Down Expand Up @@ -56,14 +57,13 @@ private void synchronizeCloudFoundryProvider(
if (!scheduledAccounts.contains(credentials.getName())) {
cloudFoundryProvider
.getAgents()
.add(
new CloudFoundryServerGroupCachingAgent(
credentials.getName(), credentials.getClient(), registry));
.add(new CloudFoundryServerGroupCachingAgent(credentials, registry));
cloudFoundryProvider
.getAgents()
.add(
new CloudFoundryLoadBalancerCachingAgent(
credentials.getName(), credentials.getClient(), registry));
.add(new CloudFoundryLoadBalancerCachingAgent(credentials, registry));
cloudFoundryProvider
.getAgents()
.add(new CloudFoundrySpaceCachingAgent(credentials, registry));
}
});
}
Expand Down
Loading

0 comments on commit 5b7470a

Please sign in to comment.