Skip to content

Commit

Permalink
feat(core): Adding pipeline template storage (#222)
Browse files Browse the repository at this point in the history
  • Loading branch information
robzienert committed Apr 26, 2017
1 parent d5f6463 commit e03b778
Show file tree
Hide file tree
Showing 12 changed files with 404 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@
import com.netflix.spinnaker.front50.model.notification.NotificationDAO;
import com.netflix.spinnaker.front50.model.pipeline.DefaultPipelineDAO;
import com.netflix.spinnaker.front50.model.pipeline.DefaultPipelineStrategyDAO;
import com.netflix.spinnaker.front50.model.pipeline.DefaultPipelineTemplateDAO;
import com.netflix.spinnaker.front50.model.pipeline.PipelineDAO;
import com.netflix.spinnaker.front50.model.pipeline.PipelineStrategyDAO;
import com.netflix.spinnaker.front50.model.pipeline.PipelineTemplateDAO;
import com.netflix.spinnaker.front50.model.project.DefaultProjectDAO;
import com.netflix.spinnaker.front50.model.project.ProjectDAO;
import com.netflix.spinnaker.front50.model.serviceaccount.DefaultServiceAccountDAO;
Expand Down Expand Up @@ -126,6 +128,18 @@ PipelineDAO pipelineDAO(StorageService storageService,
);
}

@Bean
PipelineTemplateDAO pipelineTemplateDAO(StorageService storageService,
StorageServiceConfigurationProperties storageServiceConfigurationProperties,
Registry registry) {
return new DefaultPipelineTemplateDAO(
storageService,
Schedulers.from(Executors.newFixedThreadPool(storageServiceConfigurationProperties.getPipelineTemplate().getThreadPool())),
storageServiceConfigurationProperties.getPipelineTemplate().getRefreshMs(),
registry
);
}

@Bean
SnapshotDAO snapshotDAO(StorageService storageService,
StorageServiceConfigurationProperties storageServiceConfigurationProperties,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class StorageServiceConfigurationProperties {
PerObjectType notification = new PerObjectType(20, TimeUnit.MINUTES.toMillis(1))
PerObjectType pipelineStrategy = new PerObjectType(20, TimeUnit.MINUTES.toMillis(1))
PerObjectType pipeline = new PerObjectType(20, TimeUnit.MINUTES.toMillis(1))
PerObjectType pipelineTemplate = new PerObjectType(20, TimeUnit.MINUTES.toMillis(1))
PerObjectType snapshot = new PerObjectType(2, TimeUnit.MINUTES.toMillis(1))

// not commonly used outside of Netflix
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.netflix.spinnaker.front50.model.application.Application;
import com.netflix.spinnaker.front50.model.notification.Notification;
import com.netflix.spinnaker.front50.model.pipeline.Pipeline;
import com.netflix.spinnaker.front50.model.pipeline.PipelineTemplate;
import com.netflix.spinnaker.front50.model.project.Project;
import com.netflix.spinnaker.front50.model.serviceaccount.ServiceAccount;
import com.netflix.spinnaker.front50.model.snapshot.Snapshot;
Expand All @@ -28,6 +29,7 @@ public enum ObjectType {
PROJECT(Project.class, "projects", "project-metadata.json"),
PIPELINE(Pipeline.class, "pipelines", "pipeline-metadata.json"),
STRATEGY(Pipeline.class, "pipeline-strategies", "pipeline-strategy-metadata.json"),
PIPELINE_TEMPLATE(PipelineTemplate.class, "pipeline-templates", "pipeline-template-metadata.json"),
NOTIFICATION(Notification.class, "notifications", "notification-metadata.json"),
SERVICE_ACCOUNT(ServiceAccount.class, "serviceAccounts", "serviceAccount-metadata.json"),
APPLICATION(Application.class, "applications", "application-metadata.json"),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netflix.spinnaker.front50.model.pipeline;

import com.netflix.spectator.api.Registry;
import com.netflix.spinnaker.front50.model.ObjectType;
import com.netflix.spinnaker.front50.model.StorageService;
import com.netflix.spinnaker.front50.model.StorageServiceSupport;
import org.springframework.util.Assert;
import rx.Scheduler;

import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;

public class DefaultPipelineTemplateDAO extends StorageServiceSupport<PipelineTemplate> implements PipelineTemplateDAO {

public DefaultPipelineTemplateDAO(StorageService service,
Scheduler scheduler,
long refreshIntervalMs,
Registry registry) {
super(ObjectType.PIPELINE_TEMPLATE, service, scheduler, refreshIntervalMs, registry);
}

@Override
public Collection<PipelineTemplate> getPipelineTemplatesByScope(List<String> scope) {
return all()
.stream()
.filter(pt -> pt.containsAnyScope(scope))
.collect(Collectors.toList());
}

@Override
public PipelineTemplate create(String id, PipelineTemplate item) {
Assert.notNull(item.getId(), "id field must NOT to be null!");
Assert.notEmpty(item.getScopes(), "scopes field must have at least ONE scope!");

update(id, item);
return findById(id);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netflix.spinnaker.front50.model.pipeline;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.netflix.spinnaker.front50.model.Timestamped;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class PipelineTemplate extends HashMap<String, Object> implements Timestamped {

@JsonIgnore
@SuppressWarnings("unchecked")
public List<String> getScopes() {
Map<String, Object> metadata = (Map<String, Object>) super.get("metadata");
if (metadata == null || metadata.isEmpty()) {
return Collections.emptyList();
}
return (List<String>) metadata.get("scopes");
}

@Override
public String getId() {
return (String) super.get("id");
}

@Override
public Long getLastModified() {
String updateTs = (String) super.get("updateTs");
return (updateTs != null) ? Long.valueOf(updateTs) : null;
}

@Override
public void setLastModified(Long lastModified) {
super.put("updateTs", lastModified.toString());
}

@Override
public String getLastModifiedBy() {
return (String) super.get("lastModifiedBy");
}

@Override
public void setLastModifiedBy(String lastModifiedBy) {
super.put("lastModifiedBy", lastModifiedBy);
}

public boolean containsAnyScope(List<String> scope) {
for (String s : scope) {
for (String s2 : getScopes()) {
if (s.equalsIgnoreCase(s2)) {
return true;
}
}
}
return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netflix.spinnaker.front50.model.pipeline;

import com.netflix.spinnaker.front50.model.ItemDAO;

import java.util.Collection;
import java.util.List;

public interface PipelineTemplateDAO extends ItemDAO<PipelineTemplate> {

Collection<PipelineTemplate> getPipelineTemplatesByScope(List<String> scope);
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@

import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

@Configuration
@ConditionalOnExpression("${spinnaker.gcs.enabled:false}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package com.netflix.spinnaker.front50.redis
import com.netflix.spinnaker.front50.model.application.Application
import com.netflix.spinnaker.front50.model.notification.Notification
import com.netflix.spinnaker.front50.model.pipeline.Pipeline
import com.netflix.spinnaker.front50.model.pipeline.PipelineTemplate
import com.netflix.spinnaker.front50.model.project.Project
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression
import org.springframework.boot.context.properties.EnableConfigurationProperties
Expand Down Expand Up @@ -54,6 +55,11 @@ class RedisConfig {
new RedisPipelineDAO(redisTemplate: template)
}

@Bean
RedisPipelineTemplateDAO redisPipelineTemplateDAO(RedisTemplate<String, PipelineTemplate> template) {
new RedisPipelineTemplateDAO(redisTemplate: template)
}

@Bean
RedisNotificationDAO redisNotificationDAO(RedisTemplate<String, Notification> template) {
new RedisNotificationDAO(redisTemplate: template)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netflix.spinnaker.front50.redis;

import com.google.common.collect.Lists;
import com.netflix.spinnaker.front50.exception.NotFoundException;
import com.netflix.spinnaker.front50.model.pipeline.PipelineTemplate;
import com.netflix.spinnaker.front50.model.pipeline.PipelineTemplateDAO;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.util.Assert;

import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;

public class RedisPipelineTemplateDAO implements PipelineTemplateDAO {

static final String BOOK_KEEPING_KEY = "com.netflix.spinnaker:front50:pipelineTemplates";

RedisTemplate<String, PipelineTemplate> redisTemplate;

@Override
public Collection<PipelineTemplate> getPipelineTemplatesByScope(List<String> scope) {
return all()
.stream()
.filter(it -> it.containsAnyScope(scope))
.collect(Collectors.toList());
}

@Override
public PipelineTemplate findById(String id) throws NotFoundException {
PipelineTemplate pipelineTemplate = (PipelineTemplate) redisTemplate.opsForHash().get(BOOK_KEEPING_KEY, id);
if (pipelineTemplate == null) {
throw new NotFoundException("No pipeline template found with id '" + id + "'");
}
return pipelineTemplate;
}

@Override
public Collection<PipelineTemplate> all() {
return all(true);
}

@Override
public Collection<PipelineTemplate> all(boolean refresh) {
return Lists.newArrayList(redisTemplate.opsForHash().scan(BOOK_KEEPING_KEY, ScanOptions.scanOptions().match("*").build()))
.stream()
.map(e -> (PipelineTemplate) e.getValue())
.collect(Collectors.toList());
}

@Override
public Collection<PipelineTemplate> history(String id, int maxResults) {
return Lists.newArrayList(findById(id));
}

@Override
public PipelineTemplate create(String id, PipelineTemplate item) {
Assert.notNull(item.getId(), "id field must NOT to be null!");
Assert.notEmpty(item.getScopes(), "scope field must have at least ONE scope!");

redisTemplate.opsForHash().put(BOOK_KEEPING_KEY, item.getId(), item);

return item;
}

@Override
public void update(String id, PipelineTemplate item) {
item.setLastModified(System.currentTimeMillis());
create(id, item);
}

@Override
public void delete(String id) {
redisTemplate.opsForHash().delete(BOOK_KEEPING_KEY, id);
}

@Override
public void bulkImport(Collection<PipelineTemplate> items) {
items.forEach(it -> create(it.getId(), it));
}

@Override
public boolean isHealthy() {
try {
redisTemplate.opsForHash().get("", "");
return true;
} catch (Exception e) {
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import com.netflix.spinnaker.front50.model.application.ApplicationDAO
import com.netflix.spinnaker.front50.model.application.ApplicationPermissionDAO
import com.netflix.spinnaker.front50.model.pipeline.PipelineDAO
import com.netflix.spinnaker.front50.model.pipeline.PipelineStrategyDAO
import com.netflix.spinnaker.front50.model.pipeline.PipelineTemplateDAO

import com.netflix.spinnaker.front50.model.project.ProjectDAO
import com.netflix.spinnaker.front50.model.serviceaccount.ServiceAccountDAO
import com.netflix.spinnaker.kork.web.interceptors.MetricsInterceptor
Expand Down Expand Up @@ -82,6 +84,11 @@ public class Front50WebConfig extends WebMvcConfigurerAdapter {
return new ItemDAOHealthIndicator(itemDAO: pipelineDAO)
}

@Bean
ItemDAOHealthIndicator pipelineTemplateDAOHealthIndicator(PipelineTemplateDAO pipelineTemplateDAO) {
return new ItemDAOHealthIndicator(itemDAO: pipelineTemplateDAO)
}

@Bean
ItemDAOHealthIndicator pipelineStrategyDAOHealthIndicator(PipelineStrategyDAO pipelineStrategyDAO) {
return new ItemDAOHealthIndicator(itemDAO: pipelineStrategyDAO)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package com.netflix.spinnaker.front50.controllers

import com.netflix.spinnaker.front50.model.pipeline.Pipeline
import com.netflix.spinnaker.front50.model.pipeline.PipelineDAO

import groovy.transform.InheritConstructors
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.http.HttpStatus
Expand Down

0 comments on commit e03b778

Please sign in to comment.