Skip to content

Commit

Permalink
Allow configuring max aliases for a collection (#1375)
Browse files Browse the repository at this point in the history
  • Loading branch information
jetersen committed May 24, 2020
1 parent 1dba3f8 commit 28622da
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -614,7 +614,8 @@ public void configureWith(YamlSource source) throws ConfiguratorException {

private void configureWith(List<YamlSource> sources) throws ConfiguratorException {
lastTimeLoaded = System.currentTimeMillis();
configureWith( YamlUtils.loadFrom(sources) );
ConfigurationContext context = new ConfigurationContext(registry);
configureWith(YamlUtils.loadFrom(sources, context), context);
closeSources(sources);
}

Expand All @@ -627,7 +628,8 @@ public Map<Source, String> checkWith(YamlSource source) throws ConfiguratorExcep

private Map<Source, String> checkWith(List<YamlSource> sources) throws ConfiguratorException {
if (sources.isEmpty()) return Collections.emptyMap();
return checkWith( YamlUtils.loadFrom(sources) );
ConfigurationContext context = new ConfigurationContext(registry);
return checkWith(YamlUtils.loadFrom(sources, context), context);
}

private void closeSources(List<YamlSource> sources) {
Expand Down Expand Up @@ -740,27 +742,28 @@ private static void detectVaultPluginMissing() {
}
}

private void configureWith(Mapping entries) throws ConfiguratorException {
private void configureWith(Mapping entries,
ConfigurationContext context) throws ConfiguratorException {
// Initialize secret sources
SecretSource.all().forEach(SecretSource::init);

// Check input before actually applying changes,
// so we don't let master in a weird state after some ConfiguratorException has been thrown
final Mapping clone = entries.clone();
checkWith(clone);
checkWith(clone, context);

final ObsoleteConfigurationMonitor monitor = ObsoleteConfigurationMonitor.get();
monitor.reset();
ConfigurationContext context = new ConfigurationContext(registry);
context.clearListeners();
context.addListener(monitor::record);
try (ACLContext acl = ACL.as(ACL.SYSTEM)) {
invokeWith(entries, (configurator, config) -> configurator.configure(config, context));
}
}

public Map<Source, String> checkWith(Mapping entries) throws ConfiguratorException {
public Map<Source, String> checkWith(Mapping entries,
ConfigurationContext context) throws ConfiguratorException {
Map<Source, String> issues = new HashMap<>();
ConfigurationContext context = new ConfigurationContext(registry);
context.addListener( (node,message) -> issues.put(node.getSource(), message) );
invokeWith(entries, (configurator, config) -> configurator.check(config, context));
return issues;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

import edu.umd.cs.findbugs.annotations.CheckForNull;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Util;
import io.jenkins.plugins.casc.model.CNode;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.math.NumberUtils;
import org.kohsuke.stapler.Stapler;

/**
Expand All @@ -14,9 +16,12 @@

public class ConfigurationContext implements ConfiguratorRegistry {

public static final String CASC_YAML_MAX_ALIASES_ENV = "CASC_YAML_MAX_ALIASES";
public static final String CASC_YAML_MAX_ALIASES_PROPERTY = "casc.yaml.max.aliases";
private Deprecation deprecation = Deprecation.reject;
private Restriction restriction = Restriction.reject;
private Unknown unknown = Unknown.reject;
private final int yamlMaxAliasesForCollections;

/**
* the model-introspection model to be applied by configuration-as-code.
Expand All @@ -34,12 +39,21 @@ public class ConfigurationContext implements ConfiguratorRegistry {

public ConfigurationContext(ConfiguratorRegistry registry) {
this.registry = registry;
String prop = Util.fixEmptyAndTrim(System.getProperty(
CASC_YAML_MAX_ALIASES_PROPERTY,
System.getenv(CASC_YAML_MAX_ALIASES_ENV)
));
yamlMaxAliasesForCollections = NumberUtils.toInt(prop, 50);
}

public void addListener(Listener listener) {
listeners.add(listener);
}

public void clearListeners() {
listeners.clear();
}

public void warning(@NonNull CNode node, @NonNull String message) {
for (Listener listener : listeners) {
listener.warning(node, message);
Expand Down Expand Up @@ -72,7 +86,9 @@ public void setMode(String mode) {
this.mode = mode;
}


public int getYamlMaxAliasesForCollections() {
return yamlMaxAliasesForCollections;
}

// --- delegate methods for ConfigurationContext

Expand Down
34 changes: 27 additions & 7 deletions plugin/src/main/java/io/jenkins/plugins/casc/yaml/YamlUtils.java
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package io.jenkins.plugins.casc.yaml;

import io.jenkins.plugins.casc.ConfigurationAsCode;
import io.jenkins.plugins.casc.ConfigurationContext;
import io.jenkins.plugins.casc.ConfiguratorException;
import io.jenkins.plugins.casc.model.Mapping;
import java.io.IOException;
import java.io.Reader;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Logger;
import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.composer.Composer;
import org.yaml.snakeyaml.error.YAMLException;
import org.yaml.snakeyaml.nodes.MappingNode;
import org.yaml.snakeyaml.nodes.Node;
import org.yaml.snakeyaml.nodes.NodeId;
Expand All @@ -26,12 +29,13 @@ public final class YamlUtils {

public static final Logger LOGGER = Logger.getLogger(ConfigurationAsCode.class.getName());

public static Node merge(List<YamlSource> configs) throws ConfiguratorException {
public static Node merge(List<YamlSource> configs,
ConfigurationContext context) throws ConfiguratorException {
Node root = null;
for (YamlSource source : configs) {
try (Reader r = source.read()) {

final Node node = read(source);
final Node node = read(source, context);

if (root == null) {
root = node;
Expand All @@ -48,9 +52,24 @@ public static Node merge(List<YamlSource> configs) throws ConfiguratorException
return root;
}

public static Node read(YamlSource source) throws IOException {
Composer composer = new Composer(new ParserImpl(new StreamReaderWithSource(source)), new Resolver());
return composer.getSingleNode();
public static Node read(YamlSource source, ConfigurationContext context) throws IOException {
LoaderOptions loaderOptions = new LoaderOptions();
loaderOptions.setMaxAliasesForCollections(context.getYamlMaxAliasesForCollections());
Composer composer = new Composer(
new ParserImpl(new StreamReaderWithSource(source)),
new Resolver(),
loaderOptions);
try {
return composer.getSingleNode();
} catch (YAMLException e) {
if (e.getMessage().startsWith("Number of aliases for non-scalar nodes exceeds the specified max")) {
throw new ConfiguratorException(String.format(
"%s%nYou can increase the maximum by setting an environment variable or property%n ENV: %s=\"100\"%n PROPERTY: -D%s=\"100\"",
e.getMessage(), ConfigurationContext.CASC_YAML_MAX_ALIASES_ENV,
ConfigurationContext.CASC_YAML_MAX_ALIASES_PROPERTY));
}
throw e;
}
}

private static void merge(Node root, Node node, String source) throws ConfiguratorException {
Expand Down Expand Up @@ -102,9 +121,10 @@ private static void merge(Node root, Node node, String source) throws Configurat
/**
* Load configuration-as-code model from a set of Yaml sources, merging documents
*/
public static Mapping loadFrom(List<YamlSource> sources) throws ConfiguratorException {
public static Mapping loadFrom(List<YamlSource> sources,
ConfigurationContext context) throws ConfiguratorException {
if (sources.isEmpty()) return Mapping.EMPTY;
final Node merged = merge(sources);
final Node merged = merge(sources, context);
if (merged == null) {
LOGGER.warning("configuration-as-code yaml source returned an empty document.");
return Mapping.EMPTY;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package io.jenkins.plugins.casc;

import io.jenkins.plugins.casc.misc.EnvVarsRule;
import jenkins.model.Jenkins;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import org.junit.rules.RuleChain;
import org.jvnet.hudson.test.JenkinsRule;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;

public class YamlMaxAliasesCollection {

private JenkinsRule j;

private EnvVarsRule env;

@Rule
public RuleChain rc = RuleChain.outerRule(env = new EnvVarsRule())
.around(new RestoreSystemProperties())
.around(j = new JenkinsRule());

@Test
public void testAMaxOfOneEnv() {
env.set(ConfigurationContext.CASC_YAML_MAX_ALIASES_ENV, "1");
ConfiguratorException e = assertThrows(ConfiguratorException.class, () ->
ConfigurationAsCode.get()
.configure(getClass().getResource("maxAliasesLimit.yml").toExternalForm()));
assertEquals("Number of aliases for non-scalar nodes exceeds the specified max=1\n"
+ "You can increase the maximum by setting an environment variable or property\n"
+ " ENV: CASC_YAML_MAX_ALIASES=\"100\"\n"
+ " PROPERTY: -Dcasc.yaml.max.aliases=\"100\"", e.getCause().getMessage().replaceAll("\\r\\n?", "\n"));
}

@Test
public void testAMaxOfOneProp() {
System.setProperty(ConfigurationContext.CASC_YAML_MAX_ALIASES_PROPERTY, "1");
ConfiguratorException e = assertThrows(ConfiguratorException.class, () ->
ConfigurationAsCode.get()
.configure(getClass().getResource("maxAliasesLimit.yml").toExternalForm()));
assertEquals("Number of aliases for non-scalar nodes exceeds the specified max=1\n"
+ "You can increase the maximum by setting an environment variable or property\n"
+ " ENV: CASC_YAML_MAX_ALIASES=\"100\"\n"
+ " PROPERTY: -Dcasc.yaml.max.aliases=\"100\"", e.getCause().getMessage().replaceAll("\\r\\n?", "\n"));
}

@Test
public void testAMaxOfTwoEnv() throws ConfiguratorException {
env.set(ConfigurationContext.CASC_YAML_MAX_ALIASES_ENV, "2");
ConfigurationAsCode.get().configure(getClass().getResource("maxAliasesLimit.yml").toExternalForm());
final Jenkins jenkins = Jenkins.get();
assertEquals(2, jenkins.getNodes().size());
assertEquals("static-agent1", jenkins.getNode("static-agent1").getNodeName());
assertEquals("static-agent2", jenkins.getNode("static-agent2").getNodeName());
}

@Test
public void testAMaxOfTwoProp() throws ConfiguratorException {
System.setProperty(ConfigurationContext.CASC_YAML_MAX_ALIASES_PROPERTY, "2");
ConfigurationAsCode.get().configure(getClass().getResource("maxAliasesLimit.yml").toExternalForm());
final Jenkins jenkins = Jenkins.get();
assertEquals(2, jenkins.getNodes().size());
assertEquals("static-agent1", jenkins.getNode("static-agent1").getNodeName());
assertEquals("static-agent2", jenkins.getNode("static-agent2").getNodeName());
}

@Test
public void invalidInputShouldDefaultTo50() throws ConfiguratorException {
System.setProperty(ConfigurationContext.CASC_YAML_MAX_ALIASES_PROPERTY, "HELLO");
ConfiguratorRegistry registry = ConfiguratorRegistry.get();
ConfigurationContext context = new ConfigurationContext(registry);
assertEquals(50, context.getYamlMaxAliasesForCollections());
ConfigurationAsCode.get().configure(getClass().getResource("maxAliasesLimit.yml").toExternalForm());
final Jenkins jenkins = Jenkins.get();
assertEquals(2, jenkins.getNodes().size());
assertEquals("static-agent1", jenkins.getNode("static-agent1").getNodeName());
assertEquals("static-agent2", jenkins.getNode("static-agent2").getNodeName());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
x-jenkins-linux-node: &jenkins_linux_node_anchor
remoteFS: "/home/jenkins"
launcher:
jnlp:
workDirSettings:
disabled: true
failIfWorkDirIsMissing: false
internalDir: "remoting"
workDirPath: "/tmp"
jenkins:
nodes:
- permanent:
name: "static-agent1"
<<: *jenkins_linux_node_anchor
- permanent:
name: "static-agent2"
<<: *jenkins_linux_node_anchor

0 comments on commit 28622da

Please sign in to comment.