Skip to content

Commit

Permalink
fix: Doubly braced template placeholders (${{}}) are supported
Browse files Browse the repository at this point in the history
  • Loading branch information
@manusa
manusa committed Feb 6, 2020
1 parent ddbe6ec commit f9a678d
Show file tree
Hide file tree
Showing 12 changed files with 385 additions and 163 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
### 4.7-SNAPSHOT
#### Bugs
* Fix #1847: Remove resource-\*.vm files from \*-client.jar
* Fix #959: Support for double braced `${{ }}` template placeholders

#### Improvements
* Fix #1874: Added unit tests verifying windows line-ends (CRLF) work
Expand Down
42 changes: 14 additions & 28 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/ReplaceValueStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,20 @@
*/
package io.fabric8.kubernetes.client.utils;

import io.fabric8.kubernetes.client.KubernetesClientException;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;

import static io.fabric8.kubernetes.client.utils.Utils.interpolateString;

/**
* Replaces template parameter values in the stream to avoid
* parsing issues of templates with numeric expressions
*/
public class ReplaceValueStream {
private final Map<String, String> valuesMap;
private final Map<String, String> valuesMap;

/**
* Returns a stream with the template parameter expressions replaced
Expand All @@ -36,31 +37,16 @@ public class ReplaceValueStream {
* @param valuesMap a hashmap containing parameters
* @return returns stream with template parameter expressions replaced
*/
public static InputStream replaceValues(InputStream is, Map<String, String> valuesMap) {
return new ReplaceValueStream(valuesMap).createInputStream(is);
}

public ReplaceValueStream(Map<String, String> valuesMap) {
this.valuesMap = valuesMap;
}
public static InputStream replaceValues(InputStream is, Map<String, String> valuesMap) throws IOException {
return new ReplaceValueStream(valuesMap).createInputStream(is);
}

public InputStream createInputStream(InputStream is) {
try {
String json = IOHelpers.readFully(is);
String replaced = replaceValues(json);
return new ByteArrayInputStream(replaced.getBytes());
} catch (IOException e) {
throw KubernetesClientException.launderThrowable(e);
}
}
private ReplaceValueStream(Map<String, String> valuesMap) {
this.valuesMap = valuesMap;
}

private String replaceValues(String json) {
String answer = json;
for (Map.Entry<String, String> entry : valuesMap.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
answer = Utils.replaceAllWithoutRegex(answer, "${" + key + "}", value);
}
return answer;
}
private InputStream createInputStream(InputStream is) throws IOException {
return new ByteArrayInputStream(
interpolateString(IOHelpers.readFully(is), valuesMap).getBytes(StandardCharsets.UTF_8));
}
}
29 changes: 14 additions & 15 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/Serialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public static <T> T unmarshal(InputStream is) throws KubernetesClientException {
* @throws KubernetesClientException KubernetesClientException
*/
@SuppressWarnings("unchecked")
public static <T> T unmarshal(InputStream is, Map<String, String> parameters) throws KubernetesClientException {
public static <T> T unmarshal(InputStream is, Map<String, String> parameters) {
String specFile = readSpecFileFromInputStream(is);
if (containsMultipleDocuments(specFile)) {
return (T) getKubernetesResourceList(parameters, specFile);
Expand All @@ -107,7 +107,6 @@ public static <T> T unmarshal(InputStream is, Map<String, String> parameters) th
* @param mapper The {@link ObjectMapper} to use.
* @param <T> The target type.
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, ObjectMapper mapper) {
return unmarshal(is, mapper, Collections.emptyMap());
Expand All @@ -120,11 +119,12 @@ public static <T> T unmarshal(InputStream is, ObjectMapper mapper) {
* @param parameters A {@link Map} with parameters for placeholder substitution.
* @param <T> The target type.
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, ObjectMapper mapper, Map<String, String> parameters) {
InputStream wrapped = parameters != null && !parameters.isEmpty() ? new ReplaceValueStream(parameters).createInputStream(is) : is;
try (BufferedInputStream bis = new BufferedInputStream(wrapped)) {
try (
InputStream wrapped = parameters != null && !parameters.isEmpty() ? ReplaceValueStream.replaceValues(is, parameters) : is;
BufferedInputStream bis = new BufferedInputStream(wrapped)
) {
bis.mark(-1);
int intch;
do {
Expand All @@ -147,9 +147,8 @@ public static <T> T unmarshal(InputStream is, ObjectMapper mapper, Map<String, S
* @param type The target type.
* @param <T> template argument denoting type
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static<T> T unmarshal(String str, final Class<T> type) throws KubernetesClientException {
public static<T> T unmarshal(String str, final Class<T> type) {
return unmarshal(str, type, Collections.emptyMap());
}

Expand Down Expand Up @@ -182,9 +181,8 @@ public Type getType() {
* @param type The type.
* @param <T> Template argument denoting type
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, final Class<T> type) throws KubernetesClientException {
public static <T> T unmarshal(InputStream is, final Class<T> type) {
return unmarshal(is, type, Collections.emptyMap());
}

Expand Down Expand Up @@ -213,10 +211,9 @@ public Type getType() {
* @param type The {@link TypeReference}.
* @param <T> Template argument denoting type
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, TypeReference<T> type) throws KubernetesClientException {
return unmarshal(is, type, Collections.<String, String>emptyMap());
public static <T> T unmarshal(InputStream is, TypeReference<T> type) {
return unmarshal(is, type, Collections.emptyMap());
}

/**
Expand All @@ -230,9 +227,11 @@ public static <T> T unmarshal(InputStream is, TypeReference<T> type) throws Kube
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, TypeReference<T> type, Map<String, String> parameters) throws KubernetesClientException {
InputStream wrapped = parameters != null && !parameters.isEmpty() ? new ReplaceValueStream(parameters).createInputStream(is) : is;
try (BufferedInputStream bis = new BufferedInputStream(wrapped)) {
public static <T> T unmarshal(InputStream is, TypeReference<T> type, Map<String, String> parameters) {
try (
InputStream wrapped = parameters != null && !parameters.isEmpty() ? ReplaceValueStream.replaceValues(is, parameters) : is;
BufferedInputStream bis = new BufferedInputStream(wrapped)
) {
bis.mark(-1);
int intch;
do {
Expand Down
45 changes: 43 additions & 2 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,20 @@
import java.net.URL;
import java.net.URLEncoder;
import java.nio.file.Paths;
import java.util.AbstractMap;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Stream;

import io.fabric8.kubernetes.client.KubernetesClientException;

Expand All @@ -44,6 +50,9 @@ public class Utils {
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
private static final String ALL_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789";

private Utils() {
}

public static <T> T checkNotNull(T ref, String message) {
if (ref == null) {
throw new NullPointerException(message);
Expand Down Expand Up @@ -250,8 +259,6 @@ public static String filePath(URL path) {
}
}

/**
*/
/**
* Replaces all occurrences of the from text with to text without any regular expressions
*
Expand Down Expand Up @@ -342,4 +349,38 @@ public static String getPluralFromKind(String kind) {
}
return pluralBuffer.toString();
}

/**
* Interpolates a String containing variable placeholders with the values provided in the valuesMap.
*
* <p> This method is intended to interpolate templates loaded from YAML and JSON files.
*
* <p> Placeholders are indicated by the dollar sign and curly braces ({@code ${VARIABLE_KEY}}).
*
* <p> Placeholders can also be indicated by the dollar sign and double curly braces ({@code ${{VARIABLE_KEY}}}),
* when this notation is used, the resulting value will be unquoted (if applicable), expected values should be JSON
* compatible.
*
* @see <a href="https://docs.openshift.com/container-platform/4.3/openshift_images/using-templates.html#templates-writing-parameters_using-templates">OpenShift Templates</a>
* @param valuesMap to interpolate in the String
* @param templateInput raw input containing a String with placeholders ready to be interpolated
* @return the interpolated String
*/
public static String interpolateString(String templateInput, Map<String, String> valuesMap) {
return Optional.ofNullable(valuesMap).orElse(Collections.emptyMap()).entrySet().stream()
.filter(entry -> entry.getKey() != null)
.filter(entry -> entry.getValue() != null)
.flatMap(entry -> {
final String key = entry.getKey();
final String value = entry.getValue();
return Stream.of(
new AbstractMap.SimpleEntry<>("${" + key + "}", value),
new AbstractMap.SimpleEntry<>("\"${{" + key + "}}\"", value),
new AbstractMap.SimpleEntry<>("${{" + key + "}}", value)
);
})
.map(explodedParam -> (Function<String, String>) s -> s.replace(explodedParam.getKey(), explodedParam.getValue()))
.reduce(Function.identity(), Function::andThen)
.apply(Objects.requireNonNull(templateInput, "templateInput is required"));
}
}
55 changes: 55 additions & 0 deletions kubernetes-client/src/test/java/io/fabric8/kubernetes/client/internal/UtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,13 @@
package io.fabric8.kubernetes.client.internal;

import io.fabric8.kubernetes.client.utils.Utils;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

Expand Down Expand Up @@ -55,4 +60,54 @@ public void missingEnvVarShouldReturnDefaultValue() {
assertEquals(true, Utils.getSystemPropertyOrEnvVar("DONT_EXIST", true));
}

@Test
@DisplayName("interpolateString, String with no placeholders and empty parameters, should return input")
public void interpolateStringTest() {
// Given
final String input = "I don't have placeholders";
// When
final String result = Utils.interpolateString(input, Collections.emptyMap());
// Then
assertEquals("I don't have placeholders", result);
}

@Test
@DisplayName("interpolateString, String with no placeholders and null parameters, should return input")
public void interpolateStringNullParametersTest() {
// Given
final String input = "I don't have placeholders";
// When
final String result = Utils.interpolateString(input, null);
// Then
assertEquals("I don't have placeholders", result);
}

@Test
@DisplayName("interpolateString, String with no placeholders and null parameter values, should return input")
public void interpolateStringNullParameterValuesTest() {
// Given
final String input = "I don't have placeholders";
// When
final String result = Utils.interpolateString(input, Collections.singletonMap("KEY", null));
// Then
assertEquals("I don't have placeholders", result);
}

@Test
@DisplayName("interpolateString, String with mixed placeholders and parameters, should return interpolated input")
public void interpolateStringWithParametersTest() {
// Given
final String input = "This is a \"${SINGLE_CURLY_BRACE}\" and the following is code ${NOT_REPLACED}: \"${{RENDER_UNQUOTED}}\" ${{ALREADY_UNQUOTED}}";
final Map<String, String> parameters = new HashMap<>();
parameters.put("SINGLE_CURLY_BRACE", "template string");
parameters.put("RENDER_UNQUOTED", "'1' === '1';");
parameters.put("ALREADY_UNQUOTED", "/* END */");
parameters.put("NOT_THERE", "/* END */");
parameters.put(null, "NULL key is ignored");
parameters.put("NULL_VALUE", null);
// When
final String result = Utils.interpolateString(input, parameters);
// Then
assertEquals("This is a \"template string\" and the following is code ${NOT_REPLACED}: '1' === '1'; /* END */", result);
}
}
58 changes: 41 additions & 17 deletions kubernetes-examples/src/main/java/io/fabric8/openshift/examples/TemplateExample.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,37 +24,61 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Collections;

public class TemplateExample {
private static final Logger logger = LoggerFactory.getLogger(TemplateExample.class);

public static void main(String[] args) throws IOException {
private static final String NAMESPACE = "template-example-ns";
private static final String TEST_TEMPLATE_RESOURCE = "/test-template.yml";
private static final String DEFAULT_NAME_OF_TEMPLATE = "eap6-basic-sti";

public static void main(String[] args) {
try (OpenShiftClient client = new DefaultOpenShiftClient()) {
try {
client.namespaces().createNew().withNewMetadata().withName("thisisatest").endMetadata().done();
logger.info("Creating temporary namespace '{}' for example", NAMESPACE);
client.namespaces().createNew().withNewMetadata().withName(NAMESPACE).endMetadata().done();

Template t = client.templates().load(TemplateExample.class.getResourceAsStream("/test-template.yml")).get();
for (Parameter p : t.getParameters()) {
System.out.println(p.getName());
final Template loadedTemplate = client.templates()
.load(TemplateExample.class.getResourceAsStream(TEST_TEMPLATE_RESOURCE)).get();
for (Parameter p : loadedTemplate.getParameters()) {
final String required = Boolean.TRUE.equals(p.getRequired()) ? "*" : "";
logger.info("Loaded parameter from template: {}{} - '{}' ({})",
p.getName(), required, p.getValue(), p.getGenerate());
}

t = client.templates().load(TemplateExample.class.getResourceAsStream("/test-template.yml")).get();
t = client.templates().inNamespace("thisisatest").load(TemplateExample.class.getResourceAsStream("/test-template.yml")).create();
t = client.templates().inNamespace("thisisatest").withName("eap6-basic-sti").get();
System.out.println(t.getMetadata().getName());
final Template serverUploadedTemplate = client.templates()
.inNamespace(NAMESPACE)
.load(TemplateExample.class.getResourceAsStream(TEST_TEMPLATE_RESOURCE))
.create();
logger.info("Template {} successfully created on server", serverUploadedTemplate.getMetadata().getName());
final Template serverDownloadedTemplate = client.templates().inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).get();
logger.info("Template {} successfully downloaded from server", serverDownloadedTemplate.getMetadata().getName());

final KubernetesList processedTemplateWithDefaultParameters = client.templates()
.inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).process();
logger.info("Template {} successfully processed to list with {} items, and requiredBoolean = {}",
processedTemplateWithDefaultParameters.getItems().get(0).getMetadata().getLabels().get("template"),
processedTemplateWithDefaultParameters.getItems().size(),
processedTemplateWithDefaultParameters.getItems().get(0).getMetadata().getLabels().get("requiredBoolean"));

KubernetesList l = client.templates().inNamespace("thisisatest").withName("eap6-basic-sti").process();
System.out.println(l.getItems().size());
final KubernetesList processedTemplateWithCustomParameters = client.templates()
.inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).process(Collections.singletonMap("REQUIRED_BOOLEAN", "true"));
logger.info("Template {} successfully processed to list with {} items, and requiredBoolean = {}",
processedTemplateWithCustomParameters.getItems().get(0).getMetadata().getLabels().get("template"),
processedTemplateWithCustomParameters.getItems().size(),
processedTemplateWithCustomParameters.getItems().get(0).getMetadata().getLabels().get("requiredBoolean"));

l = client.lists().load(TemplateExample.class.getResourceAsStream("/test-list.yml")).get();
System.out.println(l.getItems().size());
KubernetesList l = client.lists().load(TemplateExample.class.getResourceAsStream("/test-list.yml")).get();
logger.info("{}", l.getItems().size());

l = client.lists().inNamespace("thisisatest").load(TemplateExample.class.getResourceAsStream("/test-list.yml")).create();
final boolean templateDeleted = client.templates().inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).delete();
logger.info("Template {} was {}deleted", DEFAULT_NAME_OF_TEMPLATE, templateDeleted ? "" : "**NOT** ");
client.lists().inNamespace(NAMESPACE).load(TemplateExample.class.getResourceAsStream("/test-list.yml")).create();
} finally {
// And finally clean up the namespace
client.namespaces().withName("thisisatest").delete();
logger.info("Deleted namespace");
client.namespaces().withName(NAMESPACE).delete();
logger.info("Deleted namespace {}", NAMESPACE);
}
}
}
Expand Down
Loading

0 comments on commit f9a678d

Please sign in to comment.