Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Doubly braced template placeholders (${{}}) are supported #1977

Merged
merged 1 commit into from
Feb 8, 2020
Merged

fix: Doubly braced template placeholders (${{}}) are supported #1977

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
### 4.7-SNAPSHOT
#### Bugs
* Fix #1847: Remove resource-\*.vm files from \*-client.jar
* Fix #959: Support for double braced `${{ }}` template placeholders

#### Improvements
* Fix #1874: Added unit tests verifying windows line-ends (CRLF) work
Expand Down
42 changes: 14 additions & 28 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/ReplaceValueStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,20 @@
*/
package io.fabric8.kubernetes.client.utils;

import io.fabric8.kubernetes.client.KubernetesClientException;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;

import static io.fabric8.kubernetes.client.utils.Utils.interpolateString;

/**
* Replaces template parameter values in the stream to avoid
* parsing issues of templates with numeric expressions
*/
public class ReplaceValueStream {
private final Map<String, String> valuesMap;
private final Map<String, String> valuesMap;

/**
* Returns a stream with the template parameter expressions replaced
Expand All @@ -36,31 +37,16 @@ public class ReplaceValueStream {
* @param valuesMap a hashmap containing parameters
* @return returns stream with template parameter expressions replaced
*/
public static InputStream replaceValues(InputStream is, Map<String, String> valuesMap) {
return new ReplaceValueStream(valuesMap).createInputStream(is);
}

public ReplaceValueStream(Map<String, String> valuesMap) {
this.valuesMap = valuesMap;
}
public static InputStream replaceValues(InputStream is, Map<String, String> valuesMap) throws IOException {
return new ReplaceValueStream(valuesMap).createInputStream(is);
}

public InputStream createInputStream(InputStream is) {
try {
String json = IOHelpers.readFully(is);
String replaced = replaceValues(json);
return new ByteArrayInputStream(replaced.getBytes());
} catch (IOException e) {
throw KubernetesClientException.launderThrowable(e);
}
}
private ReplaceValueStream(Map<String, String> valuesMap) {
this.valuesMap = valuesMap;
}

private String replaceValues(String json) {
String answer = json;
for (Map.Entry<String, String> entry : valuesMap.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
answer = Utils.replaceAllWithoutRegex(answer, "${" + key + "}", value);
}
return answer;
}
private InputStream createInputStream(InputStream is) throws IOException {
return new ByteArrayInputStream(
interpolateString(IOHelpers.readFully(is), valuesMap).getBytes(StandardCharsets.UTF_8));
}
}
29 changes: 14 additions & 15 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/Serialization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public static <T> T unmarshal(InputStream is) throws KubernetesClientException {
* @throws KubernetesClientException KubernetesClientException
*/
@SuppressWarnings("unchecked")
public static <T> T unmarshal(InputStream is, Map<String, String> parameters) throws KubernetesClientException {
public static <T> T unmarshal(InputStream is, Map<String, String> parameters) {
String specFile = readSpecFileFromInputStream(is);
if (containsMultipleDocuments(specFile)) {
return (T) getKubernetesResourceList(parameters, specFile);
Expand All @@ -107,7 +107,6 @@ public static <T> T unmarshal(InputStream is, Map<String, String> parameters) th
* @param mapper The {@link ObjectMapper} to use.
* @param <T> The target type.
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, ObjectMapper mapper) {
return unmarshal(is, mapper, Collections.emptyMap());
Expand All @@ -120,11 +119,12 @@ public static <T> T unmarshal(InputStream is, ObjectMapper mapper) {
* @param parameters A {@link Map} with parameters for placeholder substitution.
* @param <T> The target type.
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, ObjectMapper mapper, Map<String, String> parameters) {
InputStream wrapped = parameters != null && !parameters.isEmpty() ? new ReplaceValueStream(parameters).createInputStream(is) : is;
try (BufferedInputStream bis = new BufferedInputStream(wrapped)) {
try (
InputStream wrapped = parameters != null && !parameters.isEmpty() ? ReplaceValueStream.replaceValues(is, parameters) : is;
BufferedInputStream bis = new BufferedInputStream(wrapped)
) {
bis.mark(-1);
int intch;
do {
Expand All @@ -147,9 +147,8 @@ public static <T> T unmarshal(InputStream is, ObjectMapper mapper, Map<String, S
* @param type The target type.
* @param <T> template argument denoting type
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static<T> T unmarshal(String str, final Class<T> type) throws KubernetesClientException {
public static<T> T unmarshal(String str, final Class<T> type) {
return unmarshal(str, type, Collections.emptyMap());
}

Expand Down Expand Up @@ -182,9 +181,8 @@ public Type getType() {
* @param type The type.
* @param <T> Template argument denoting type
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, final Class<T> type) throws KubernetesClientException {
public static <T> T unmarshal(InputStream is, final Class<T> type) {
return unmarshal(is, type, Collections.emptyMap());
}

Expand Down Expand Up @@ -213,10 +211,9 @@ public Type getType() {
* @param type The {@link TypeReference}.
* @param <T> Template argument denoting type
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, TypeReference<T> type) throws KubernetesClientException {
return unmarshal(is, type, Collections.<String, String>emptyMap());
public static <T> T unmarshal(InputStream is, TypeReference<T> type) {
return unmarshal(is, type, Collections.emptyMap());
}

/**
Expand All @@ -230,9 +227,11 @@ public static <T> T unmarshal(InputStream is, TypeReference<T> type) throws Kube
* @return returns de-serialized object
* @throws KubernetesClientException KubernetesClientException
*/
public static <T> T unmarshal(InputStream is, TypeReference<T> type, Map<String, String> parameters) throws KubernetesClientException {
InputStream wrapped = parameters != null && !parameters.isEmpty() ? new ReplaceValueStream(parameters).createInputStream(is) : is;
try (BufferedInputStream bis = new BufferedInputStream(wrapped)) {
public static <T> T unmarshal(InputStream is, TypeReference<T> type, Map<String, String> parameters) {
try (
InputStream wrapped = parameters != null && !parameters.isEmpty() ? ReplaceValueStream.replaceValues(is, parameters) : is;
BufferedInputStream bis = new BufferedInputStream(wrapped)
) {
bis.mark(-1);
int intch;
do {
Expand Down
45 changes: 43 additions & 2 deletions kubernetes-client/src/main/java/io/fabric8/kubernetes/client/utils/Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,20 @@
import java.net.URL;
import java.net.URLEncoder;
import java.nio.file.Paths;
import java.util.AbstractMap;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Stream;

import io.fabric8.kubernetes.client.KubernetesClientException;

Expand All @@ -44,6 +50,9 @@ public class Utils {
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
private static final String ALL_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789";

private Utils() {
}

public static <T> T checkNotNull(T ref, String message) {
if (ref == null) {
throw new NullPointerException(message);
Expand Down Expand Up @@ -250,8 +259,6 @@ public static String filePath(URL path) {
}
}

/**
*/
/**
* Replaces all occurrences of the from text with to text without any regular expressions
*
Expand Down Expand Up @@ -342,4 +349,38 @@ public static String getPluralFromKind(String kind) {
}
return pluralBuffer.toString();
}

/**
* Interpolates a String containing variable placeholders with the values provided in the valuesMap.
*
* <p> This method is intended to interpolate templates loaded from YAML and JSON files.
*
* <p> Placeholders are indicated by the dollar sign and curly braces ({@code ${VARIABLE_KEY}}).
*
* <p> Placeholders can also be indicated by the dollar sign and double curly braces ({@code ${{VARIABLE_KEY}}}),
* when this notation is used, the resulting value will be unquoted (if applicable), expected values should be JSON
* compatible.
*
* @see <a href="https://docs.openshift.com/container-platform/4.3/openshift_images/using-templates.html#templates-writing-parameters_using-templates">OpenShift Templates</a>
* @param valuesMap to interpolate in the String
* @param templateInput raw input containing a String with placeholders ready to be interpolated
* @return the interpolated String
*/
public static String interpolateString(String templateInput, Map<String, String> valuesMap) {
return Optional.ofNullable(valuesMap).orElse(Collections.emptyMap()).entrySet().stream()
.filter(entry -> entry.getKey() != null)
.filter(entry -> entry.getValue() != null)
.flatMap(entry -> {
final String key = entry.getKey();
final String value = entry.getValue();
return Stream.of(
new AbstractMap.SimpleEntry<>("${" + key + "}", value),
new AbstractMap.SimpleEntry<>("\"${{" + key + "}}\"", value),
new AbstractMap.SimpleEntry<>("${{" + key + "}}", value)
);
})
.map(explodedParam -> (Function<String, String>) s -> s.replace(explodedParam.getKey(), explodedParam.getValue()))
.reduce(Function.identity(), Function::andThen)
.apply(Objects.requireNonNull(templateInput, "templateInput is required"));
}
}
55 changes: 55 additions & 0 deletions kubernetes-client/src/test/java/io/fabric8/kubernetes/client/internal/UtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,13 @@
package io.fabric8.kubernetes.client.internal;

import io.fabric8.kubernetes.client.utils.Utils;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

Expand Down Expand Up @@ -55,4 +60,54 @@ public void missingEnvVarShouldReturnDefaultValue() {
assertEquals(true, Utils.getSystemPropertyOrEnvVar("DONT_EXIST", true));
}

@Test
@DisplayName("interpolateString, String with no placeholders and empty parameters, should return input")
public void interpolateStringTest() {
// Given
final String input = "I don't have placeholders";
// When
final String result = Utils.interpolateString(input, Collections.emptyMap());
// Then
assertEquals("I don't have placeholders", result);
}

@Test
@DisplayName("interpolateString, String with no placeholders and null parameters, should return input")
public void interpolateStringNullParametersTest() {
// Given
final String input = "I don't have placeholders";
// When
final String result = Utils.interpolateString(input, null);
// Then
assertEquals("I don't have placeholders", result);
}

@Test
@DisplayName("interpolateString, String with no placeholders and null parameter values, should return input")
public void interpolateStringNullParameterValuesTest() {
// Given
final String input = "I don't have placeholders";
// When
final String result = Utils.interpolateString(input, Collections.singletonMap("KEY", null));
// Then
assertEquals("I don't have placeholders", result);
}

@Test
@DisplayName("interpolateString, String with mixed placeholders and parameters, should return interpolated input")
public void interpolateStringWithParametersTest() {
// Given
final String input = "This is a \"${SINGLE_CURLY_BRACE}\" and the following is code ${NOT_REPLACED}: \"${{RENDER_UNQUOTED}}\" ${{ALREADY_UNQUOTED}}";
final Map<String, String> parameters = new HashMap<>();
parameters.put("SINGLE_CURLY_BRACE", "template string");
parameters.put("RENDER_UNQUOTED", "'1' === '1';");
parameters.put("ALREADY_UNQUOTED", "/* END */");
parameters.put("NOT_THERE", "/* END */");
parameters.put(null, "NULL key is ignored");
parameters.put("NULL_VALUE", null);
// When
final String result = Utils.interpolateString(input, parameters);
// Then
assertEquals("This is a \"template string\" and the following is code ${NOT_REPLACED}: '1' === '1'; /* END */", result);
}
}
58 changes: 41 additions & 17 deletions kubernetes-examples/src/main/java/io/fabric8/openshift/examples/TemplateExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,37 +24,61 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Collections;

public class TemplateExample {
private static final Logger logger = LoggerFactory.getLogger(TemplateExample.class);

public static void main(String[] args) throws IOException {
private static final String NAMESPACE = "template-example-ns";
private static final String TEST_TEMPLATE_RESOURCE = "/test-template.yml";
private static final String DEFAULT_NAME_OF_TEMPLATE = "eap6-basic-sti";

public static void main(String[] args) {
try (OpenShiftClient client = new DefaultOpenShiftClient()) {
try {
client.namespaces().createNew().withNewMetadata().withName("thisisatest").endMetadata().done();
logger.info("Creating temporary namespace '{}' for example", NAMESPACE);
client.namespaces().createNew().withNewMetadata().withName(NAMESPACE).endMetadata().done();

Template t = client.templates().load(TemplateExample.class.getResourceAsStream("/test-template.yml")).get();
for (Parameter p : t.getParameters()) {
System.out.println(p.getName());
final Template loadedTemplate = client.templates()
.load(TemplateExample.class.getResourceAsStream(TEST_TEMPLATE_RESOURCE)).get();
for (Parameter p : loadedTemplate.getParameters()) {
final String required = Boolean.TRUE.equals(p.getRequired()) ? "*" : "";
logger.info("Loaded parameter from template: {}{} - '{}' ({})",
p.getName(), required, p.getValue(), p.getGenerate());
}

t = client.templates().load(TemplateExample.class.getResourceAsStream("/test-template.yml")).get();
t = client.templates().inNamespace("thisisatest").load(TemplateExample.class.getResourceAsStream("/test-template.yml")).create();
t = client.templates().inNamespace("thisisatest").withName("eap6-basic-sti").get();
System.out.println(t.getMetadata().getName());
final Template serverUploadedTemplate = client.templates()
.inNamespace(NAMESPACE)
.load(TemplateExample.class.getResourceAsStream(TEST_TEMPLATE_RESOURCE))
.create();
logger.info("Template {} successfully created on server", serverUploadedTemplate.getMetadata().getName());
final Template serverDownloadedTemplate = client.templates().inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).get();
logger.info("Template {} successfully downloaded from server", serverDownloadedTemplate.getMetadata().getName());

final KubernetesList processedTemplateWithDefaultParameters = client.templates()
.inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).process();
logger.info("Template {} successfully processed to list with {} items, and requiredBoolean = {}",
processedTemplateWithDefaultParameters.getItems().get(0).getMetadata().getLabels().get("template"),
processedTemplateWithDefaultParameters.getItems().size(),
processedTemplateWithDefaultParameters.getItems().get(0).getMetadata().getLabels().get("requiredBoolean"));

KubernetesList l = client.templates().inNamespace("thisisatest").withName("eap6-basic-sti").process();
System.out.println(l.getItems().size());
final KubernetesList processedTemplateWithCustomParameters = client.templates()
.inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).process(Collections.singletonMap("REQUIRED_BOOLEAN", "true"));
logger.info("Template {} successfully processed to list with {} items, and requiredBoolean = {}",
processedTemplateWithCustomParameters.getItems().get(0).getMetadata().getLabels().get("template"),
processedTemplateWithCustomParameters.getItems().size(),
processedTemplateWithCustomParameters.getItems().get(0).getMetadata().getLabels().get("requiredBoolean"));

l = client.lists().load(TemplateExample.class.getResourceAsStream("/test-list.yml")).get();
System.out.println(l.getItems().size());
KubernetesList l = client.lists().load(TemplateExample.class.getResourceAsStream("/test-list.yml")).get();
logger.info("{}", l.getItems().size());

l = client.lists().inNamespace("thisisatest").load(TemplateExample.class.getResourceAsStream("/test-list.yml")).create();
final boolean templateDeleted = client.templates().inNamespace(NAMESPACE).withName(DEFAULT_NAME_OF_TEMPLATE).delete();
logger.info("Template {} was {}deleted", DEFAULT_NAME_OF_TEMPLATE, templateDeleted ? "" : "**NOT** ");
client.lists().inNamespace(NAMESPACE).load(TemplateExample.class.getResourceAsStream("/test-list.yml")).create();
} finally {
// And finally clean up the namespace
client.namespaces().withName("thisisatest").delete();
logger.info("Deleted namespace");
client.namespaces().withName(NAMESPACE).delete();
logger.info("Deleted namespace {}", NAMESPACE);
}
}
}
Expand Down
Loading