Skip to content

Commit

Permalink
feat(kubernetes,google): Support retrieving config files from config …
Browse files Browse the repository at this point in the history
…server. (#3812)
  • Loading branch information
scottfrederick authored and Jammy Louie committed Jun 27, 2019
1 parent 66496ac commit b04520b
Show file tree
Hide file tree
Showing 14 changed files with 446 additions and 82 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package com.netflix.spinnaker.clouddriver.appengine.security
import com.netflix.spinnaker.cats.module.CatsModule
import com.netflix.spinnaker.clouddriver.appengine.AppengineJobExecutor
import com.netflix.spinnaker.clouddriver.appengine.config.AppengineConfigurationProperties
import com.netflix.spinnaker.clouddriver.data.ConfigFileService
import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository
import com.netflix.spinnaker.clouddriver.security.ProviderUtils
import groovy.util.logging.Slf4j
Expand All @@ -34,21 +35,24 @@ class AppengineCredentialsInitializer {
String clouddriverUserAgentApplicationName,
AppengineConfigurationProperties appengineConfigurationProperties,
AccountCredentialsRepository accountCredentialsRepository,
AppengineJobExecutor jobExecutor) {
AppengineJobExecutor jobExecutor,
ConfigFileService configFileService) {

synchronizeAppengineAccounts(clouddriverUserAgentApplicationName,
appengineConfigurationProperties,
null,
accountCredentialsRepository,
jobExecutor)
jobExecutor,
configFileService)
}

private List<? extends AppengineNamedAccountCredentials> synchronizeAppengineAccounts(
String clouddriverUserAgentApplicationName,
AppengineConfigurationProperties appengineConfigurationProperties,
CatsModule catsModule,
AccountCredentialsRepository accountCredentialsRepository,
AppengineJobExecutor jobExecutor) {
AppengineJobExecutor jobExecutor,
ConfigFileService configFileService) {

def (ArrayList<AppengineConfigurationProperties.ManagedAccount> accountsToAdd, List<String> namesOfDeletedAccounts) =
ProviderUtils.calculateAccountDeltas(accountCredentialsRepository,
Expand All @@ -63,7 +67,7 @@ class AppengineCredentialsInitializer {
}
managedAccount.initialize(jobExecutor, gcloudPath)

def jsonKey = AppengineCredentialsInitializer.getJsonKey(managedAccount)
def jsonKey = configFileService.getContents(managedAccount.getJsonPath())
def appengineAccount = new AppengineNamedAccountCredentials.Builder()
.name(managedAccount.name)
.environment(managedAccount.environment ?: managedAccount.name)
Expand Down Expand Up @@ -104,10 +108,4 @@ class AppengineCredentialsInitializer {
it instanceof AppengineNamedAccountCredentials
} as List<AppengineNamedAccountCredentials>
}

private static String getJsonKey(AppengineConfigurationProperties.ManagedAccount managedAccount) {
def inputStream = managedAccount.inputStream

inputStream ? new String(inputStream.bytes) : null
}
}
1 change: 0 additions & 1 deletion clouddriver-cloudfoundry/clouddriver-cloudfoundry.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,4 @@ dependencies {
testImplementation "org.junit.jupiter:junit-jupiter-engine"
testImplementation "org.junit.jupiter:junit-jupiter-params"
testImplementation "org.mockito:mockito-core"
testImplementation "org.mockito:mockito-core"
}
6 changes: 6 additions & 0 deletions clouddriver-core/clouddriver-core.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ dependencies {
implementation "org.codehaus.groovy:groovy-all"
implementation "org.springframework.boot:spring-boot-actuator"
implementation "org.springframework.cloud:spring-cloud-context"
implementation "org.springframework.cloud:spring-cloud-config-server"
implementation "redis.clients:jedis"

testImplementation project(":cats:cats-test")
Expand All @@ -40,4 +41,9 @@ dependencies {
testImplementation "org.objenesis:objenesis"
testImplementation "org.spockframework:spock-core"
testImplementation "org.spockframework:spock-spring"
testImplementation "org.assertj:assertj-core"
testImplementation "org.junit.jupiter:junit-jupiter-api"
testImplementation "org.junit.jupiter:junit-jupiter-engine"
testImplementation "org.junit.jupiter:junit-jupiter-params"
testImplementation "org.mockito:mockito-core"
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,16 @@
package com.netflix.spinnaker.clouddriver.cache;

import com.netflix.spinnaker.cats.cluster.DefaultAgentIntervalProvider;
import com.netflix.spinnaker.clouddriver.data.ConfigFileService;
import com.netflix.spinnaker.clouddriver.refresh.CloudConfigRefreshScheduler;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.cloud.config.server.EnableConfigServer;
import org.springframework.cloud.config.server.environment.EnvironmentRepository;
import org.springframework.cloud.config.server.resource.ResourceRepository;
import org.springframework.cloud.context.refresh.ContextRefresher;
import org.springframework.context.annotation.*;
import org.springframework.core.type.AnnotatedTypeMetadata;
Expand All @@ -33,21 +37,40 @@
*/
@Configuration
@AutoConfigureAfter({RedisCacheConfig.class, DynomiteCacheConfig.class})
@Conditional(RemoteConfigSourceConfigured.class)
public class CloudConfigRefreshConfig {

@Bean
@ConditionalOnBean(DefaultAgentIntervalProvider.class)
public CloudConfigRefreshScheduler intervalProviderConfigRefreshScheduler(
ContextRefresher contextRefresher, DefaultAgentIntervalProvider agentIntervalProvider) {
return new CloudConfigRefreshScheduler(contextRefresher, agentIntervalProvider.getInterval());
@Configuration
@Conditional(RemoteConfigSourceConfigured.class)
@EnableConfigServer
static class RemoteConfigSourceConfiguration {
@Bean
ConfigFileService configFileService(
ResourceRepository resourceRepository, EnvironmentRepository environmentRepository) {
return new ConfigFileService(resourceRepository, environmentRepository);
}

@Bean
@ConditionalOnBean(DefaultAgentIntervalProvider.class)
public CloudConfigRefreshScheduler intervalProviderConfigRefreshScheduler(
ContextRefresher contextRefresher, DefaultAgentIntervalProvider agentIntervalProvider) {
return new CloudConfigRefreshScheduler(contextRefresher, agentIntervalProvider.getInterval());
}

@Bean
@ConditionalOnMissingBean(DefaultAgentIntervalProvider.class)
public CloudConfigRefreshScheduler defaultIntervalConfigRefreshScheduler(
ContextRefresher contextRefresher) {
return new CloudConfigRefreshScheduler(contextRefresher, 60);
}
}

@Bean
@ConditionalOnMissingBean(DefaultAgentIntervalProvider.class)
public CloudConfigRefreshScheduler defaultIntervalConfigRefreshScheduler(
ContextRefresher contextRefresher) {
return new CloudConfigRefreshScheduler(contextRefresher, 60);
@Configuration
static class DefaultConfiguration {
@Bean
@ConditionalOnMissingBean(ConfigFileService.class)
ConfigFileService configFileService() {
return new ConfigFileService();
}
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
/*
* Copyright 2019 Pivotal, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.clouddriver.data;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.config.environment.Environment;
import org.springframework.cloud.config.server.environment.EnvironmentRepository;
import org.springframework.cloud.config.server.resource.NoSuchResourceException;
import org.springframework.cloud.config.server.resource.ResourceRepository;
import org.springframework.cloud.config.server.support.EnvironmentPropertySource;
import org.springframework.core.env.StandardEnvironment;
import org.springframework.core.io.Resource;

@Slf4j
public class ConfigFileService {
private static final String CONFIG_SERVER_RESOURCE_PREFIX = "configserver:";
private static final String CLASSPATH_FILE_PREFIX = "classpath:";

private final ResourceRepository resourceRepository;
private final EnvironmentRepository environmentRepository;

@Value("${spring.application.name:application}")
private String applicationName = "application";

public ConfigFileService() {
resourceRepository = null;
environmentRepository = null;
}

public ConfigFileService(
ResourceRepository resourceRepository, EnvironmentRepository environmentRepository) {
this.resourceRepository = resourceRepository;
this.environmentRepository = environmentRepository;
}

public String getLocalPath(String path, String tempFilePrefix, String tempFileSuffix) {
if (StringUtils.isNotEmpty(path)) {
if (isCloudConfigResource(path)) {
String configServerContents = retrieveFromConfigServer(path);
return writeToTempFile(configServerContents, tempFilePrefix, tempFileSuffix);
} else {
return verifyLocalPath(path);
}
}

return null;
}

public String getLocalPathForContents(
String contents, String tempFilePrefix, String tempFileSuffix) {
if (StringUtils.isNotEmpty(contents)) {
return writeToTempFile(contents, tempFilePrefix, tempFileSuffix);
}

return null;
}

public String getContents(String path) {
if (StringUtils.isNotEmpty(path)) {
if (isCloudConfigResource(path)) {
return retrieveFromConfigServer(path);
}

if (isClasspathResource(path)) {
return retrieveFromClasspath(path);
}

return retrieveFromLocalPath(path);
}

return null;
}

private boolean isCloudConfigResource(String path) {
return path.startsWith(CONFIG_SERVER_RESOURCE_PREFIX);
}

private boolean isClasspathResource(String path) {
return path.startsWith(CLASSPATH_FILE_PREFIX);
}

private String verifyLocalPath(String path) {
if (Files.isReadable(Paths.get(path))) {
return path;
} else {
throw new RuntimeException("File \"" + path + "\" not found or is not readable");
}
}

private String retrieveFromLocalPath(String path) {
try {
Path filePath = Paths.get(path);
return new String(Files.readAllBytes(filePath));
} catch (FileNotFoundException e) {
throw new RuntimeException("File \"" + path + "\" not found or is not readable", e);
} catch (IOException e) {
throw new RuntimeException("Exception reading file " + path, e);
}
}

private String retrieveFromClasspath(String path) {
try {
String filePath = path.substring(CLASSPATH_FILE_PREFIX.length());
InputStream inputStream = getClass().getResourceAsStream(filePath);
return IOUtils.toString(inputStream, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new RuntimeException("Exception reading classpath resource \"" + path + "\"", e);
}
}

private String retrieveFromConfigServer(String path) {
if (resourceRepository == null || environmentRepository == null) {
throw new RuntimeException(
"Config Server repository not configured for resource \"" + path + "\"");
}

try {
String fileName = path.substring(CONFIG_SERVER_RESOURCE_PREFIX.length());
Resource resource = this.resourceRepository.findOne(applicationName, null, null, fileName);
try (InputStream inputStream = resource.getInputStream()) {
Environment environment = this.environmentRepository.findOne(applicationName, null, null);

String text = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
StandardEnvironment preparedEnvironment =
EnvironmentPropertySource.prepareEnvironment(environment);
return EnvironmentPropertySource.resolvePlaceholders(preparedEnvironment, text);
}
} catch (NoSuchResourceException e) {
throw new RuntimeException("The resource \"" + path + "\" was not found in config server", e);
} catch (IOException e) {
throw new RuntimeException("Exception reading config server resource \"" + path + "\"", e);
}
}

private String writeToTempFile(String contents, String tempFilePrefix, String tempFileSuffix) {
try {
File tempFile = File.createTempFile(tempFilePrefix, tempFileSuffix);
tempFile.deleteOnExit();

BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile));
writer.write(contents);
writer.close();

log.info(
"Configuration for {} written to temporary file {}",
tempFilePrefix,
tempFile.getAbsolutePath());

return tempFile.getAbsolutePath();
} catch (IOException e) {
throw new RuntimeException(
"Exception writing temporary file with prefix \""
+ tempFilePrefix
+ "\": "
+ e.getMessage(),
e);
}
}
}
Loading

0 comments on commit b04520b

Please sign in to comment.