Skip to content

Commit

Permalink
Easy by-account separation of config and objects (#86)
Browse files Browse the repository at this point in the history 
feat(persistence): Add CONFIGURATION_STORE to allow configs to be stored in a different account than intermediate results.
  • Loading branch information
Michael Graff authored and Matt Duftler committed Sep 18, 2017
1 parent 46e708b commit 1863ce6
Show file tree
Hide file tree
Showing 7 changed files with 92 additions and 61 deletions.
12 changes: 8 additions & 4 deletions kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/orca/AtlasFetchTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public TaskResult execute(Stage stage) {
Map<String, Object> context = stage.getContext();
String metricsAccountName = (String)context.get("metricsAccountName");
String storageAccountName = (String)context.get("storageAccountName");
String configurationAccountName = (String)context.get("configurationAccountName");
String canaryConfigId = (String)context.get("canaryConfigId");
AtlasCanaryScope atlasCanaryScope =
objectMapper.convertValue(stage.getContext().get("atlasCanaryScope"), AtlasCanaryScope.class);
Expand All @@ -93,14 +94,17 @@ public TaskResult execute(Stage stage) {
String resolvedStorageAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName,
AccountCredentials.Type.OBJECT_STORE,
accountCredentialsRepository);
StorageService storageService =
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);
StorageService configurationService =
storageServiceRepository
.getOne(resolvedStorageAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to load canary config."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to load canary config."));

try {
CanaryConfig canaryConfig =
storageService.loadObject(resolvedStorageAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());
configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());

List<String> metricSetListIds = synchronousQueryProcessor.processQuery(resolvedMetricsAccountName,
resolvedStorageAccountName,
Expand Down
21 changes: 15 additions & 6 deletions kayenta-core/src/main/java/com/netflix/kayenta/canary/orca/CanaryJudgeTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,21 +73,30 @@ public long getTimeout() {
public TaskResult execute(Stage stage) {
Map<String, Object> context = stage.getContext();
String storageAccountName = (String)context.get("storageAccountName");
String resolvedAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName,
AccountCredentials.Type.OBJECT_STORE,
accountCredentialsRepository);
String resolvedStorageAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName,
AccountCredentials.Type.OBJECT_STORE,
accountCredentialsRepository);
String configurationAccountName = (String)context.get("configurationAccountName");
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);
String canaryConfigId = (String)context.get("canaryConfigId");
String metricSetPairListId = (String)context.get("metricSetPairListId");
Map<String, String> orchestratorScoreThresholdsMap = (Map<String, String>)context.get("orchestratorScoreThresholds");
CanaryClassifierThresholdsConfig orchestratorScoreThresholds = objectMapper.convertValue(orchestratorScoreThresholdsMap,
CanaryClassifierThresholdsConfig.class);
StorageService storageService =
storageServiceRepository
.getOne(resolvedAccountName)
.getOne(resolvedStorageAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to load metric set lists."));

CanaryConfig canaryConfig = storageService.loadObject(resolvedAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());
List<MetricSetPair> metricSetPairList = storageService.loadObject(resolvedAccountName, ObjectType.METRIC_SET_PAIR_LIST, metricSetPairListId);
StorageService configurationService =
storageServiceRepository
.getOne(resolvedStorageAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to load configurations."));

CanaryConfig canaryConfig = configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());
List<MetricSetPair> metricSetPairList = storageService.loadObject(resolvedStorageAccountName, ObjectType.METRIC_SET_PAIR_LIST, metricSetPairListId);
CanaryJudgeConfig canaryJudgeConfig = canaryConfig.getJudge();
CanaryJudge canaryJudge = null;

Expand Down
3 changes: 2 additions & 1 deletion kayenta-core/src/main/java/com/netflix/kayenta/security/AccountCredentials.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public interface AccountCredentials<T> {

enum Type {
METRICS_STORE,
OBJECT_STORE
OBJECT_STORE,
CONFIGURATION_STORE
}
}
13 changes: 9 additions & 4 deletions ...nta-prometheus/src/main/java/com/netflix/kayenta/prometheus/orca/PrometheusFetchTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public TaskResult execute(Stage stage) {
Map<String, Object> context = stage.getContext();
String metricsAccountName = (String)context.get("metricsAccountName");
String storageAccountName = (String)context.get("storageAccountName");
String configurationAccountName = (String)context.get("configurationAccountName");
String canaryConfigId = (String)context.get("canaryConfigId");
PrometheusCanaryScope prometheusCanaryScope =
objectMapper.convertValue(stage.getContext().get("prometheusCanaryScope"), PrometheusCanaryScope.class);
Expand All @@ -83,14 +84,18 @@ public TaskResult execute(Stage stage) {
String resolvedStorageAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName,
AccountCredentials.Type.OBJECT_STORE,
accountCredentialsRepository);
StorageService storageService =
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);

StorageService configurationService =
storageServiceRepository
.getOne(resolvedStorageAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to load canary config."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to load canary config."));

try {
CanaryConfig canaryConfig =
storageService.loadObject(resolvedStorageAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());
configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());


Instant startTimeInstant = Instant.parse(prometheusCanaryScope.getIntervalStartTimeIso());
Expand Down
13 changes: 9 additions & 4 deletions ...-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/orca/StackdriverFetchTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ public TaskResult execute(Stage stage) {
Map<String, Object> context = stage.getContext();
String metricsAccountName = (String)context.get("metricsAccountName");
String storageAccountName = (String)context.get("storageAccountName");
String configurationAccountName = (String)context.get("configurationAccountName");
String canaryConfigId = (String)context.get("canaryConfigId");
StackdriverCanaryScope stackdriverCanaryScope =
objectMapper.convertValue(stage.getContext().get("stackdriverCanaryScope"), StackdriverCanaryScope.class);
Expand All @@ -81,14 +82,18 @@ public TaskResult execute(Stage stage) {
String resolvedStorageAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName,
AccountCredentials.Type.OBJECT_STORE,
accountCredentialsRepository);
StorageService storageService =
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);

StorageService configurationService =
storageServiceRepository
.getOne(resolvedStorageAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to load canary config."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to load canary config."));

try {
CanaryConfig canaryConfig =
storageService.loadObject(resolvedStorageAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());
configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());

Instant startTimeInstant = Instant.parse(stackdriverCanaryScope.getIntervalStartTimeIso());
long startTimeMillis = startTimeInstant.toEpochMilli();
Expand Down
75 changes: 37 additions & 38 deletions kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryConfigController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.regex.Pattern;

Expand All @@ -59,32 +58,32 @@ public class CanaryConfigController {

@ApiOperation(value = "Retrieve a canary config from object storage")
@RequestMapping(value = "/{canaryConfigId:.+}", method = RequestMethod.GET)
public CanaryConfig loadCanaryConfig(@RequestParam(required = false) final String accountName,
public CanaryConfig loadCanaryConfig(@RequestParam(required = false) final String configurationAccountName,
@PathVariable String canaryConfigId) {
String resolvedAccountName = CredentialsHelper.resolveAccountByNameOrType(accountName,
AccountCredentials.Type.OBJECT_STORE,
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);
StorageService storageService =
StorageService configurationService =
storageServiceRepository
.getOne(resolvedAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to read canary config from bucket."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to read canary config from bucket."));

canaryConfigId = canaryConfigId.toLowerCase();

return storageService.loadObject(resolvedAccountName, ObjectType.CANARY_CONFIG, canaryConfigId);
return configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId);
}

@ApiOperation(value = "Write a canary config to object storage")
@RequestMapping(consumes = "application/json", method = RequestMethod.POST)
public String storeCanaryConfig(@RequestParam(required = false) final String accountName,
public String storeCanaryConfig(@RequestParam(required = false) final String configurationAccountName,
@RequestBody CanaryConfig canaryConfig) throws IOException {
String resolvedAccountName = CredentialsHelper.resolveAccountByNameOrType(accountName,
AccountCredentials.Type.OBJECT_STORE,
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);
StorageService storageService =
StorageService configurationService =
storageServiceRepository
.getOne(resolvedAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to write canary config to bucket."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to write canary config to bucket."));

if (canaryConfig.getCreatedTimestamp() == null) {
canaryConfig.setCreatedTimestamp(System.currentTimeMillis());
Expand Down Expand Up @@ -116,9 +115,9 @@ public String storeCanaryConfig(@RequestParam(required = false) final String acc
}

try {
storageService.loadObject(resolvedAccountName, ObjectType.CANARY_CONFIG, canaryConfigId);
configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId);
} catch (IllegalArgumentException e) {
storageService.storeObject(resolvedAccountName, ObjectType.CANARY_CONFIG, canaryConfigId, canaryConfig);
configurationService.storeObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId, canaryConfig);

return canaryConfigId;
}
Expand All @@ -128,16 +127,16 @@ public String storeCanaryConfig(@RequestParam(required = false) final String acc

@ApiOperation(value = "Update a canary config")
@RequestMapping(value = "/{canaryConfigId:.+}", consumes = "application/json", method = RequestMethod.PUT)
public String updateCanaryConfig(@RequestParam(required = false) final String accountName,
public String updateCanaryConfig(@RequestParam(required = false) final String configurationAccountName,
@PathVariable String canaryConfigId,
@RequestBody CanaryConfig canaryConfig) throws IOException {
String resolvedAccountName = CredentialsHelper.resolveAccountByNameOrType(accountName,
AccountCredentials.Type.OBJECT_STORE,
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);
StorageService storageService =
StorageService configurationService =
storageServiceRepository
.getOne(resolvedAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to write canary config to bucket."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to write canary config to bucket."));

canaryConfig.setUpdatedTimestamp(System.currentTimeMillis());
canaryConfig.setUpdatedTimestampIso(Instant.ofEpochMilli(canaryConfig.getUpdatedTimestamp()).toString());
Expand All @@ -149,44 +148,44 @@ public String updateCanaryConfig(@RequestParam(required = false) final String ac
canaryConfigId = canaryConfigId.toLowerCase();

try {
storageService.loadObject(resolvedAccountName, ObjectType.CANARY_CONFIG, canaryConfigId);
configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId);
} catch (Exception e) {
throw new IllegalArgumentException("Canary config '" + canaryConfigId + "' does not exist.");
}

storageService.storeObject(resolvedAccountName, ObjectType.CANARY_CONFIG, canaryConfigId, canaryConfig);
configurationService.storeObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId, canaryConfig);
return canaryConfigId;
}

@ApiOperation(value = "Delete a canary config")
@RequestMapping(value = "/{canaryConfigId:.+}", method = RequestMethod.DELETE)
public void deleteCanaryConfig(@RequestParam(required = false) final String accountName,
public void deleteCanaryConfig(@RequestParam(required = false) final String configurationAccountName,
@PathVariable String canaryConfigId,
HttpServletResponse response) {
String resolvedAccountName = CredentialsHelper.resolveAccountByNameOrType(accountName,
AccountCredentials.Type.OBJECT_STORE,
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);
StorageService storageService =
StorageService configurationService =
storageServiceRepository
.getOne(resolvedAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to delete canary config."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to delete canary config."));

storageService.deleteObject(resolvedAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());
configurationService.deleteObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId.toLowerCase());

response.setStatus(HttpStatus.NO_CONTENT.value());
}

@ApiOperation(value = "Retrieve a list of canary config ids and timestamps")
@RequestMapping(method = RequestMethod.GET)
public List<Map<String, Object>> listAllCanaryConfigs(@RequestParam(required = false) final String accountName) {
String resolvedAccountName = CredentialsHelper.resolveAccountByNameOrType(accountName,
AccountCredentials.Type.OBJECT_STORE,
public List<Map<String, Object>> listAllCanaryConfigs(@RequestParam(required = false) final String configurationAccountName) {
String resolvedConfigurationAccountName = CredentialsHelper.resolveAccountByNameOrType(configurationAccountName,
AccountCredentials.Type.CONFIGURATION_STORE,
accountCredentialsRepository);
StorageService storageService =
StorageService configurationService =
storageServiceRepository
.getOne(resolvedAccountName)
.orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to list all canary configs."));
.getOne(resolvedConfigurationAccountName)
.orElseThrow(() -> new IllegalArgumentException("No configuration service was configured; unable to list all canary configs."));

return storageService.listObjectKeys(resolvedAccountName, ObjectType.CANARY_CONFIG);
return configurationService.listObjectKeys(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG);
}
}
Loading

0 comments on commit 1863ce6

Please sign in to comment.