From ce5c216d84b4f13245b14929ca33272980e408fd Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Tue, 12 Feb 2019 11:53:56 -0500 Subject: [PATCH 01/10] generate parameterized configs for handlers - #216, #217 --- light-rest-4j/pom.xml | 4 + .../codegen/rest/OpenApiGenerator.java | 4 + .../codegen/rest/YAMLFileParameterizer.java | 178 ++++++++++++++++++ .../src/main/resources/config/audit.yml | 55 ++++++ .../src/main/resources/config/body.yml | 3 + .../src/main/resources/config/correlation.yml | 7 + .../src/main/resources/config/health.yml | 5 + .../src/main/resources/config/info.yml | 5 + .../src/main/resources/config/metrics.yml | 20 ++ .../resources/config/openapi-validator.yml | 6 + .../src/main/resources/config/sanitizer.yml | 11 ++ .../main/resources/config/traceability.yml | 4 + .../codegen/OpenApiGeneratorTest.java | 40 +++- light-rest-4j/src/test/resources/config.json | 3 +- 14 files changed, 334 insertions(+), 11 deletions(-) create mode 100644 light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java create mode 100644 light-rest-4j/src/main/resources/config/audit.yml create mode 100644 light-rest-4j/src/main/resources/config/body.yml create mode 100644 light-rest-4j/src/main/resources/config/correlation.yml create mode 100644 light-rest-4j/src/main/resources/config/health.yml create mode 100644 light-rest-4j/src/main/resources/config/info.yml create mode 100644 light-rest-4j/src/main/resources/config/metrics.yml create mode 100644 light-rest-4j/src/main/resources/config/openapi-validator.yml create mode 100644 light-rest-4j/src/main/resources/config/sanitizer.yml create mode 100644 light-rest-4j/src/main/resources/config/traceability.yml diff --git a/light-rest-4j/pom.xml b/light-rest-4j/pom.xml index 476109830..5b47ae19e 100644 --- a/light-rest-4j/pom.xml +++ b/light-rest-4j/pom.xml @@ -67,6 +67,10 @@ com.thoughtworks.qdox qdox + + org.yaml + snakeyaml + ch.qos.logback logback-classic diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java index 8f94abb19..7bae007ea 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java @@ -107,6 +107,8 @@ public void generate(String targetPath, Object model, Any config) throws IOExcep enableParamDescription = config.toBoolean("enableParamDescription"); generateValuesYml = config.toBoolean("generateValuesYml"); + + boolean generateEnvVars = config.toBoolean("generateEnvVars"); String version = config.toString("version").trim(); String serviceId = config.get("groupId").toString().trim() + "." + config.get("artifactId").toString().trim() + "-" + config.get("version").toString().trim(); @@ -173,6 +175,8 @@ public void generate(String targetPath, Object model, Any config) throws IOExcep // values.yml file, transfer only if explicitly set in the config.json if(generateValuesYml) transfer(targetPath, ("src.main.resources.config").replace(".", separator), "values.yml", templates.rest.openapi.values.template()); + + YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_SOURCE_DIR, targetPath+separator+YAMLFileParameterizer.DEFAULT_SOURCE_DIR, generateEnvVars); } } diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java new file mode 100644 index 000000000..4bec9148d --- /dev/null +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java @@ -0,0 +1,178 @@ +package com.networknt.codegen.rest; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.composer.Composer; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.parser.ParserImpl; +import org.yaml.snakeyaml.reader.StreamReader; +import org.yaml.snakeyaml.resolver.Resolver; + +public class YAMLFileParameterizer { + private static final Logger logger = LoggerFactory.getLogger(YAMLFileParameterizer.class); + private static final String KEY_SEPARATOR = "."; + private static final String YML_EXT = ".yml"; + + public static final String DEFAULT_SOURCE_DIR = "src/main/resources/config/"; + + public static void rewriteAll(String sourceDir, String destDir, boolean generateEnvVars) { + rewriteAll(new File(sourceDir), new File(destDir), generateEnvVars); + } + + public static void rewriteAll(File sourceDir, File destDir, boolean generateEnvVars) { + + if (!sourceDir.exists() || !sourceDir.isDirectory()) { + logger.error("{} does not exist or is not a folder.", sourceDir); + return; + } + + + if (!destDir.isDirectory() || !destDir.exists()) { + if (!destDir.mkdir()) { + logger.error("Failed to create dir {}", destDir); + return; + } + } + + File[] files = sourceDir.listFiles((dir, name)->name.toLowerCase().endsWith(YML_EXT)); + + for (File file: files) { + rewrite(file, new File(destDir, file.getName()), generateEnvVars); + } + } + + public static void rewrite(String sourceFilename, String destFilename, boolean generateEnvVars) { + rewrite(new File(sourceFilename), new File(destFilename), generateEnvVars); + } + + public static void rewrite(File srcFile, File destFile, boolean generateEnvVars) { + if (!generateEnvVars) { + copy(srcFile, destFile); + }else { + parameterize(srcFile, destFile); + } + } + + protected static void copy(File srcFile, File destFile) { + if (!srcFile.exists()) { + logger.error("File {} does not exist.", srcFile.getAbsolutePath()); + return; + } + + if (srcFile.exists()) { + try { + FileUtils.copyFile(srcFile, destFile); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + } + + protected static void parameterize(File srcFile, File destFile) { + if (!srcFile.exists()) { + logger.error("File {} does not exist.", srcFile.getAbsolutePath()); + return; + } + + try { + Composer composer = new Composer(new ParserImpl(new StreamReader(new FileReader(srcFile))), new Resolver()); + + Node document = composer.getSingleNode(); + + if (null==document) { + logger.warn("Cannot parse the YAML file. The file {} may be empty or invalid.", srcFile.getAbsolutePath()); + return; + } + + String prefix = stripExtension(srcFile.getName()); + + if (document instanceof MappingNode) { + List srclines = FileUtils.readLines(srcFile, (String)null); + + List destlines = parameterize(prefix, srclines, (MappingNode)document); + + FileUtils.writeLines(destFile, destlines); + }else { + throw new UnsupportedNodeTypeException(document.getClass().getCanonicalName()); + } + + + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + + protected static List parameterize(String prefix, List srclines, MappingNode node) { + List destlines = new ArrayList<>(); + + List tuples = node.getValue(); + + int pos = 0; + + for (NodeTuple tuple: tuples) { + Node k = tuple.getKeyNode(); + Node v = tuple.getValueNode(); + + int startLine = k.getStartMark().getLine(); + int stopLine = v.getEndMark().getLine(); + + copy(srclines, destlines, pos, startLine); + + pos = stopLine+1; + + if (k instanceof ScalarNode) { + ScalarNode sk = (ScalarNode)k; + + if (v instanceof ScalarNode) { + ScalarNode sv = (ScalarNode)v; + + destlines.add(String.format("%s:${%s%s%s:%s}", sk.getValue(), prefix, KEY_SEPARATOR, sk.getValue(), sv.getValue())); + }else { + destlines.add(String.format("%s:${%s%s%s}", sk.getValue(), prefix, KEY_SEPARATOR, sk.getValue())); + } + }else { + throw new UnsupportedNodeTypeException(k.getClass().getCanonicalName()); + } + } + + return destlines; + } + + protected static void copy(List srclines, List destlines, int start, int end) { + if (end>=srclines.size()) { + throw new IndexOutOfBoundsException(); + } + + for (int i=start; i= 400 +# log level is controlled by logLevel +auditOnError: false + +# log level; by default set to info +logLevelIsError: false + + +# Output header elements. You can add more if you want. +headers: + +# Correlation Id +- X-Correlation-Id + +# Traceability Id +- X-Traceability-Id + +# Output from id token and access token +audit: + +# Client Id +- client_id + +# User Id in id token, this is optional +- user_id + +# Client Id in scope/access token, this is optional +- scope_client_id + +# Request endpoint uri@method. +- endpoint + +# Service ID assigned to the service, this is optional and must be set by the service in its implementation +- serviceId + +# Request Body, this is optional and must be set by the service in its implementation +- request + +# Response payload, this is optional and must be set by the service in its implementation +- response diff --git a/light-rest-4j/src/main/resources/config/body.yml b/light-rest-4j/src/main/resources/config/body.yml new file mode 100644 index 000000000..a6ec2f041 --- /dev/null +++ b/light-rest-4j/src/main/resources/config/body.yml @@ -0,0 +1,3 @@ +--- +# Enable body parse flag +enabled: true diff --git a/light-rest-4j/src/main/resources/config/correlation.yml b/light-rest-4j/src/main/resources/config/correlation.yml new file mode 100644 index 000000000..66871e9de --- /dev/null +++ b/light-rest-4j/src/main/resources/config/correlation.yml @@ -0,0 +1,7 @@ +# Correlation Id Handler Configuration +--- +# If enabled is true, the handler will be injected into the request and response chain. +enabled: true + +# If set to true, it will auto-generate the correlationID if it is not provided in the request +autogenCorrelationID: true diff --git a/light-rest-4j/src/main/resources/config/health.yml b/light-rest-4j/src/main/resources/config/health.yml new file mode 100644 index 000000000..999e05a1d --- /dev/null +++ b/light-rest-4j/src/main/resources/config/health.yml @@ -0,0 +1,5 @@ +--- +# Server health endpoint that can output OK to indicate the server is alive + +# true to enable this middleware handler. +enabled: true diff --git a/light-rest-4j/src/main/resources/config/info.yml b/light-rest-4j/src/main/resources/config/info.yml new file mode 100644 index 000000000..6d0a5d0e0 --- /dev/null +++ b/light-rest-4j/src/main/resources/config/info.yml @@ -0,0 +1,5 @@ +--- +# Server info endpoint that can output environment and component along with configuration + +# Indicate if the server info is enable or not. +enableServerInfo: true diff --git a/light-rest-4j/src/main/resources/config/metrics.yml b/light-rest-4j/src/main/resources/config/metrics.yml new file mode 100644 index 000000000..83d1e56b2 --- /dev/null +++ b/light-rest-4j/src/main/resources/config/metrics.yml @@ -0,0 +1,20 @@ +--- +# Metrics handler configuration + +# If metrics handler is enabled or not +enabled: false + +# influxdb protocal can be http, https +influxdbProtocol: http +# influxdb hostname +influxdbHost: localhost +# influxdb port number +influxdbPort: 8086 +# influxdb database name +influxdbName: metrics +# influxdb user +influxdbUser: admin +# influx db password +influxdbPass: admin +# report and reset metrics in minutes. +reportInMinutes: 1 diff --git a/light-rest-4j/src/main/resources/config/openapi-validator.yml b/light-rest-4j/src/main/resources/config/openapi-validator.yml new file mode 100644 index 000000000..717bd7e83 --- /dev/null +++ b/light-rest-4j/src/main/resources/config/openapi-validator.yml @@ -0,0 +1,6 @@ +# Default openapi-validator configuration +--- +# Enable request validation. Response validation is not done on the server but client. +enabled: true +# Log error message if validation error occurs +logError: true diff --git a/light-rest-4j/src/main/resources/config/sanitizer.yml b/light-rest-4j/src/main/resources/config/sanitizer.yml new file mode 100644 index 000000000..8bbb75b0d --- /dev/null +++ b/light-rest-4j/src/main/resources/config/sanitizer.yml @@ -0,0 +1,11 @@ +--- +# Sanitize request for cross site scripting during runtime + +# indicate if sanitizer is enabled or not +enabled: true + +# if it is enabled, does body need to be sanitized +sanitizeBody: true + +# if it is enabled, does header need to be sanitized +sanitizeHeader: false diff --git a/light-rest-4j/src/main/resources/config/traceability.yml b/light-rest-4j/src/main/resources/config/traceability.yml new file mode 100644 index 000000000..ea193e1ca --- /dev/null +++ b/light-rest-4j/src/main/resources/config/traceability.yml @@ -0,0 +1,4 @@ +# Traceability Id handler configuration +--- +# Indicate if this handler is enabled or not +enabled: true diff --git a/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java b/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java index 3bec377a6..adc92327f 100644 --- a/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java +++ b/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java @@ -1,17 +1,9 @@ package com.networknt.codegen; -import com.jsoniter.JsonIterator; -import com.jsoniter.any.Any; -import com.networknt.codegen.rest.OpenApiGenerator; -import com.thoughtworks.qdox.JavaProjectBuilder; -import com.thoughtworks.qdox.model.JavaClass; -import com.thoughtworks.qdox.model.JavaField; -import com.thoughtworks.qdox.model.JavaPackage; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; @@ -19,6 +11,19 @@ import java.util.List; import java.util.Scanner; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.jsoniter.JsonIterator; +import com.jsoniter.any.Any; +import com.networknt.codegen.rest.OpenApiGenerator; +import com.networknt.codegen.rest.YAMLFileParameterizer; +import com.thoughtworks.qdox.JavaProjectBuilder; +import com.thoughtworks.qdox.model.JavaClass; +import com.thoughtworks.qdox.model.JavaField; +import com.thoughtworks.qdox.model.JavaPackage; + /** * @author Steve Hu */ @@ -133,4 +138,19 @@ public void testInvalidVaribleNameGeneratorJson() throws IOException { } } } + + @Test + public void testYAMLFileParameterizer() { + File srcDir = new File("src/main/resources/config/"); + File destDir = new File("/tmp/test/"); + + FilenameFilter filter = (dir, name)->name.toLowerCase().endsWith(".yml"); + + YAMLFileParameterizer.rewriteAll(srcDir, destDir, true); + + int srcCount = srcDir.list(filter).length; + int destCount = destDir.list(filter).length; + + assertTrue(srcCount==destCount); + } } diff --git a/light-rest-4j/src/test/resources/config.json b/light-rest-4j/src/test/resources/config.json index 0d34ea488..85d0fedbd 100644 --- a/light-rest-4j/src/test/resources/config.json +++ b/light-rest-4j/src/test/resources/config.json @@ -24,5 +24,6 @@ }, "supportH2ForTest": false, "supportClient": false, - "dockerOrganization": "networknt" + "dockerOrganization": "networknt", + "generateEnvVars": true } From 1ccdefd52d7c49504ede3340395696731c3f3d9c Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Tue, 12 Feb 2019 12:14:00 -0500 Subject: [PATCH 02/10] clean up --- .../networknt/codegen/rest/YAMLFileParameterizer.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java index 4bec9148d..c89b058dc 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java @@ -70,12 +70,10 @@ protected static void copy(File srcFile, File destFile) { return; } - if (srcFile.exists()) { - try { - FileUtils.copyFile(srcFile, destFile); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } + try { + FileUtils.copyFile(srcFile, destFile); + } catch (IOException e) { + logger.error(e.getMessage(), e); } } From 4059b9b28c26504100eb902e787c40dcaa2732cb Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Tue, 12 Feb 2019 14:18:26 -0500 Subject: [PATCH 03/10] make the config 'generateEnvVars' optional - #216, #217 --- .../codegen/rest/OpenApiGenerator.java | 46 +++++++++++++------ light-rest-4j/src/test/resources/config.json | 3 +- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java index fed7cc717..a483cc586 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java @@ -1,15 +1,6 @@ package com.networknt.codegen.rest; -import com.jsoniter.JsonIterator; -import com.jsoniter.ValueType; -import com.jsoniter.any.Any; -import com.jsoniter.output.JsonStream; -import com.networknt.codegen.Generator; -import com.networknt.codegen.Utils; -import com.networknt.jsonoverlay.Overlay; -import com.networknt.oas.OpenApiParser; -import com.networknt.oas.model.*; -import com.networknt.oas.model.impl.OpenApi3Impl; +import static java.io.File.separator; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -20,11 +11,33 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.stream.Collectors; -import static java.io.File.separator; +import com.jsoniter.JsonIterator; +import com.jsoniter.ValueType; +import com.jsoniter.any.Any; +import com.jsoniter.output.JsonStream; +import com.networknt.codegen.Generator; +import com.networknt.codegen.Utils; +import com.networknt.jsonoverlay.Overlay; +import com.networknt.oas.OpenApiParser; +import com.networknt.oas.model.Example; +import com.networknt.oas.model.MediaType; +import com.networknt.oas.model.OpenApi3; +import com.networknt.oas.model.Operation; +import com.networknt.oas.model.Parameter; +import com.networknt.oas.model.Path; +import com.networknt.oas.model.Response; +import com.networknt.oas.model.Schema; +import com.networknt.oas.model.Server; +import com.networknt.oas.model.impl.OpenApi3Impl; /** * The input for OpenAPI 3.0 generator include config with json format @@ -38,6 +51,8 @@ public class OpenApiGenerator implements Generator { private Map typeMapping = new HashMap<>(); + static final String GENERATE_ENV_VARS="generateEnvVars"; + // optional generation parameters. if not set, they use default values as boolean prometheusMetrics =false; boolean skipHealthCheck = false; @@ -108,7 +123,8 @@ public void generate(String targetPath, Object model, Any config) throws IOExcep generateValuesYml = config.toBoolean("generateValuesYml"); - boolean generateEnvVars = config.toBoolean("generateEnvVars"); + Any generateEnvVarsConfig = config.get("generateEnvVars"); + String version = config.toString("version").trim(); String serviceId = config.get("groupId").toString().trim() + "." + config.get("artifactId").toString().trim() + "-" + config.get("version").toString().trim(); @@ -176,7 +192,9 @@ public void generate(String targetPath, Object model, Any config) throws IOExcep if(generateValuesYml) transfer(targetPath, ("src.main.resources.config").replace(".", separator), "values.yml", templates.rest.openapi.values.template()); - YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_SOURCE_DIR, targetPath+separator+YAMLFileParameterizer.DEFAULT_SOURCE_DIR, generateEnvVars); + if (config.keys().contains(GENERATE_ENV_VARS)) { + YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_SOURCE_DIR, targetPath+separator+YAMLFileParameterizer.DEFAULT_SOURCE_DIR, config.get(GENERATE_ENV_VARS).toBoolean()); + } } } diff --git a/light-rest-4j/src/test/resources/config.json b/light-rest-4j/src/test/resources/config.json index 85d0fedbd..0d34ea488 100644 --- a/light-rest-4j/src/test/resources/config.json +++ b/light-rest-4j/src/test/resources/config.json @@ -24,6 +24,5 @@ }, "supportH2ForTest": false, "supportClient": false, - "dockerOrganization": "networknt", - "generateEnvVars": true + "dockerOrganization": "networknt" } From a7161a6030c5c314101cea55bd8ed9ebfdc6060e Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Tue, 12 Feb 2019 14:20:58 -0500 Subject: [PATCH 04/10] clean up --- .../main/java/com/networknt/codegen/rest/OpenApiGenerator.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java index a483cc586..7c868ad66 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java @@ -123,9 +123,6 @@ public void generate(String targetPath, Object model, Any config) throws IOExcep generateValuesYml = config.toBoolean("generateValuesYml"); - Any generateEnvVarsConfig = config.get("generateEnvVars"); - - String version = config.toString("version").trim(); String serviceId = config.get("groupId").toString().trim() + "." + config.get("artifactId").toString().trim() + "-" + config.get("version").toString().trim(); From bdccc6133e5967a7163c6f69a96fb4565c2fe118 Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Wed, 13 Feb 2019 11:28:23 -0500 Subject: [PATCH 05/10] fix file loading defect - #216, #217 --- .../codegen/rest/OpenApiGenerator.java | 4 +- .../codegen/rest/YAMLFileParameterizer.java | 237 ++++++++++++++---- .../{config => handlerconfig}/audit.yml | 0 .../{config => handlerconfig}/body.yml | 0 .../{config => handlerconfig}/correlation.yml | 0 .../{config => handlerconfig}/health.yml | 0 .../{config => handlerconfig}/info.yml | 0 .../{config => handlerconfig}/metrics.yml | 0 .../openapi-validator.yml | 0 .../{config => handlerconfig}/sanitizer.yml | 0 .../traceability.yml | 0 .../codegen/OpenApiGeneratorTest.java | 38 ++- light-rest-4j/src/test/resources/config.json | 7 +- 13 files changed, 227 insertions(+), 59 deletions(-) rename light-rest-4j/src/main/resources/{config => handlerconfig}/audit.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/body.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/correlation.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/health.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/info.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/metrics.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/openapi-validator.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/sanitizer.yml (100%) rename light-rest-4j/src/main/resources/{config => handlerconfig}/traceability.yml (100%) diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java index 7c868ad66..97320d60b 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java @@ -51,7 +51,7 @@ public class OpenApiGenerator implements Generator { private Map typeMapping = new HashMap<>(); - static final String GENERATE_ENV_VARS="generateEnvVars"; + public static final String GENERATE_ENV_VARS="generateEnvVars"; // optional generation parameters. if not set, they use default values as boolean prometheusMetrics =false; @@ -190,7 +190,7 @@ public void generate(String targetPath, Object model, Any config) throws IOExcep transfer(targetPath, ("src.main.resources.config").replace(".", separator), "values.yml", templates.rest.openapi.values.template()); if (config.keys().contains(GENERATE_ENV_VARS)) { - YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_SOURCE_DIR, targetPath+separator+YAMLFileParameterizer.DEFAULT_SOURCE_DIR, config.get(GENERATE_ENV_VARS).toBoolean()); + YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_RESOURCE_LOCATION, targetPath+separator+YAMLFileParameterizer.DEFAULT_DEST_DIR, config.get(GENERATE_ENV_VARS).asMap()); } } } diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java index c89b058dc..f01470652 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java @@ -1,12 +1,20 @@ package com.networknt.codegen.rest; +import java.io.BufferedReader; import java.io.File; +import java.io.FileOutputStream; import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,54 +27,144 @@ import org.yaml.snakeyaml.reader.StreamReader; import org.yaml.snakeyaml.resolver.Resolver; +import com.jsoniter.any.Any; + public class YAMLFileParameterizer { private static final Logger logger = LoggerFactory.getLogger(YAMLFileParameterizer.class); private static final String KEY_SEPARATOR = "."; private static final String YML_EXT = ".yml"; - public static final String DEFAULT_SOURCE_DIR = "src/main/resources/config/"; + public static final String CLASS_PATH_PREFIX="classpath:"; + public static final String DEFAULT_RESOURCE_LOCATION = CLASS_PATH_PREFIX+"handlerconfig/"; + public static final String DEFAULT_DEST_DIR = "src/main/resources/config"; + + protected static final String KEY_GENERATE="generate"; + protected static final String KEY_SKIP_ARRAY="skipArray"; + protected static final String KEY_SKIP_MAP="skipMap"; - public static void rewriteAll(String sourceDir, String destDir, boolean generateEnvVars) { - rewriteAll(new File(sourceDir), new File(destDir), generateEnvVars); + public static void rewriteAll(String srcLocation, String destDir, Map generateEnvVars) { + if (fromClasspath(srcLocation)) { + rewriteResources(resolveLocation(srcLocation), destDir, generateEnvVars); + }else { + rewriteFiles(new File(srcLocation), new File(destDir), generateEnvVars); + } } - public static void rewriteAll(File sourceDir, File destDir, boolean generateEnvVars) { - + public static void rewriteFiles(File sourceDir, File destDir, Map generateEnvVars) { if (!sourceDir.exists() || !sourceDir.isDirectory()) { logger.error("{} does not exist or is not a folder.", sourceDir); return; } - - + if (!destDir.isDirectory() || !destDir.exists()) { if (!destDir.mkdir()) { logger.error("Failed to create dir {}", destDir); return; } } - + File[] files = sourceDir.listFiles((dir, name)->name.toLowerCase().endsWith(YML_EXT)); - + for (File file: files) { - rewrite(file, new File(destDir, file.getName()), generateEnvVars); + rewriteFile(file, new File(destDir, file.getName()), generateEnvVars); + } + } + + + public static void rewriteResources(String resourceLocation, String destDir, Map generateEnvVars) { + try { + List filenames = IOUtils.readLines(YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation), (String)null); + + List ymlFileNames = filenames.stream().filter(name->name.toLowerCase().endsWith(YML_EXT)).collect(Collectors.toList()); + + File dest = new File(destDir); + + if (!dest.isDirectory() || !dest.exists()) { + if (!dest.mkdir()) { + logger.error("Failed to create dir {}", destDir); + return; + } + } + + for (String filename: ymlFileNames) { + rewriteResource(stripExtension(filename), resourceLocation+File.separator+filename, destDir+File.separator+filename, generateEnvVars); + } + + + } catch (IOException e) { + logger.error(e.getMessage(), e); } } - public static void rewrite(String sourceFilename, String destFilename, boolean generateEnvVars) { - rewrite(new File(sourceFilename), new File(destFilename), generateEnvVars); + public static void rewriteResource(String filename, String resourceLocation, String destFilename, Map generateEnvVars) { + if (logger.isDebugEnabled()) { + logger.debug("rewriting resource {}", resourceLocation); + } + + if (!getValue(generateEnvVars, KEY_GENERATE)) { + copyResource(resourceLocation, destFilename); + }else { + Node document = loadResource(resourceLocation); + List lines = readResource(resourceLocation); + + if (null!=document && null!=lines) { + parameterize(filename, document, lines, new File(destFilename), generateEnvVars); + } + } } - public static void rewrite(File srcFile, File destFile, boolean generateEnvVars) { - if (!generateEnvVars) { - copy(srcFile, destFile); + public static void rewriteFile(File srcFile, File destFile, Map generateEnvVars) { + if (logger.isDebugEnabled()) { + logger.debug("rewriting file {}", srcFile.getAbsolutePath()); + } + + if (!getValue(generateEnvVars, KEY_GENERATE)) { + copyFile(srcFile, destFile); }else { - parameterize(srcFile, destFile); + Node document = loadFile(srcFile); + List lines = readFile(srcFile); + + String filename = stripExtension(srcFile.getName()); + + if (null!=document && null!=lines) { + parameterize(filename, document, lines, destFile, generateEnvVars); + } + } + } + + protected static boolean fromClasspath(String location) { + return StringUtils.trimToEmpty(location).toLowerCase().startsWith(CLASS_PATH_PREFIX); + } + + protected static String resolveLocation(String location) { + if (StringUtils.isNotBlank(location) && location.contains(CLASS_PATH_PREFIX)) { + return StringUtils.trimToEmpty(location).substring(CLASS_PATH_PREFIX.length()); + } + + return location; + } + + protected static boolean getValue(Map generateEnvVars, String key) { + if (generateEnvVars.containsKey(key)) { + return generateEnvVars.get(key).toBoolean(); + } + + return false; + } + + protected static void copyResource(String resourceLocation, String destFilename) { + try (InputStream in = YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation); + FileOutputStream out = new FileOutputStream(destFilename)) { + IOUtils.copy(in, out); + } catch (IOException e) { + logger.error(e.getMessage(), e); } } - protected static void copy(File srcFile, File destFile) { + protected static void copyFile(File srcFile, File destFile) { if (!srcFile.exists()) { - logger.error("File {} does not exist.", srcFile.getAbsolutePath()); + logger.error("The file {} cannot be found", srcFile.getAbsolutePath()); + return; } @@ -77,45 +175,73 @@ protected static void copy(File srcFile, File destFile) { } } - protected static void parameterize(File srcFile, File destFile) { - if (!srcFile.exists()) { - logger.error("File {} does not exist.", srcFile.getAbsolutePath()); - return; + protected static Node loadResource(String resourceLocation) { + try (InputStream in = YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation); + BufferedReader inputReader = new BufferedReader(new InputStreamReader(in))) { + Composer composer = new Composer(new ParserImpl(new StreamReader(inputReader)), new Resolver()); + return composer.getSingleNode(); + } catch (Exception e) { + logger.error(e.getMessage(), e); } - + + return null; + } + + protected static List readResource(String resourceLocation) { + try (InputStream in = YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation)) { + return IOUtils.readLines(in, (String) null); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + protected static Node loadFile(File file) { + try (FileReader inputReader = new FileReader(file)) { + Composer composer = new Composer(new ParserImpl(new StreamReader(inputReader)), new Resolver()); + return composer.getSingleNode(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + protected static List readFile(File file) { try { - Composer composer = new Composer(new ParserImpl(new StreamReader(new FileReader(srcFile))), new Resolver()); - - Node document = composer.getSingleNode(); - - if (null==document) { - logger.warn("Cannot parse the YAML file. The file {} may be empty or invalid.", srcFile.getAbsolutePath()); - return; - } - - String prefix = stripExtension(srcFile.getName()); - + return FileUtils.readLines(file, (String) null); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + protected static void parameterize(String filename, Node document, List srclines, File destFile, Map generateEnvVars) { + try (FileWriter writer = new FileWriter(destFile)) { if (document instanceof MappingNode) { - List srclines = FileUtils.readLines(srcFile, (String)null); - - List destlines = parameterize(prefix, srclines, (MappingNode)document); - + List destlines = parameterize(filename, srclines, (MappingNode) document, generateEnvVars); + FileUtils.writeLines(destFile, destlines); - }else { + + } else { throw new UnsupportedNodeTypeException(document.getClass().getCanonicalName()); } - - + } catch (IOException e) { logger.error(e.getMessage(), e); } } - protected static List parameterize(String prefix, List srclines, MappingNode node) { + protected static List parameterize(String filename, List srclines, MappingNode node, Map generateEnvVars) { List destlines = new ArrayList<>(); List tuples = node.getValue(); + boolean skipArray = getValue(generateEnvVars, KEY_SKIP_ARRAY); + boolean skipMap = getValue(generateEnvVars, KEY_SKIP_MAP); + int pos = 0; for (NodeTuple tuple: tuples) { @@ -125,20 +251,30 @@ protected static List parameterize(String prefix, List srclines, int startLine = k.getStartMark().getLine(); int stopLine = v.getEndMark().getLine(); - copy(srclines, destlines, pos, startLine); - - pos = stopLine+1; - if (k instanceof ScalarNode) { ScalarNode sk = (ScalarNode)k; if (v instanceof ScalarNode) { ScalarNode sv = (ScalarNode)v; - destlines.add(String.format("%s:${%s%s%s:%s}", sk.getValue(), prefix, KEY_SEPARATOR, sk.getValue(), sv.getValue())); + copy(srclines, destlines, pos, startLine); + destlines.add(String.format("%s:${%s%s%s:%s}", sk.getValue(), filename, KEY_SEPARATOR, sk.getValue(), sv.getValue())); + + pos = stopLine+1; }else { - destlines.add(String.format("%s:${%s%s%s}", sk.getValue(), prefix, KEY_SEPARATOR, sk.getValue())); - } + if (!skipArray && !skipMap) { + copy(srclines, destlines, pos, startLine); + destlines.add(String.format("%s:${%s%s%s}", sk.getValue(), filename, KEY_SEPARATOR, sk.getValue())); + + pos = stopLine+1; + }else { + copy(srclines, destlines, pos, stopLine); + + pos = stopLine; + } + } + + }else { throw new UnsupportedNodeTypeException(k.getClass().getCanonicalName()); } @@ -148,7 +284,8 @@ protected static List parameterize(String prefix, List srclines, } protected static void copy(List srclines, List destlines, int start, int end) { - if (end>=srclines.size()) { + if (end>srclines.size()) { + logger.error("attemp to read line {}. total length {}", end, srclines.size()); throw new IndexOutOfBoundsException(); } diff --git a/light-rest-4j/src/main/resources/config/audit.yml b/light-rest-4j/src/main/resources/handlerconfig/audit.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/audit.yml rename to light-rest-4j/src/main/resources/handlerconfig/audit.yml diff --git a/light-rest-4j/src/main/resources/config/body.yml b/light-rest-4j/src/main/resources/handlerconfig/body.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/body.yml rename to light-rest-4j/src/main/resources/handlerconfig/body.yml diff --git a/light-rest-4j/src/main/resources/config/correlation.yml b/light-rest-4j/src/main/resources/handlerconfig/correlation.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/correlation.yml rename to light-rest-4j/src/main/resources/handlerconfig/correlation.yml diff --git a/light-rest-4j/src/main/resources/config/health.yml b/light-rest-4j/src/main/resources/handlerconfig/health.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/health.yml rename to light-rest-4j/src/main/resources/handlerconfig/health.yml diff --git a/light-rest-4j/src/main/resources/config/info.yml b/light-rest-4j/src/main/resources/handlerconfig/info.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/info.yml rename to light-rest-4j/src/main/resources/handlerconfig/info.yml diff --git a/light-rest-4j/src/main/resources/config/metrics.yml b/light-rest-4j/src/main/resources/handlerconfig/metrics.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/metrics.yml rename to light-rest-4j/src/main/resources/handlerconfig/metrics.yml diff --git a/light-rest-4j/src/main/resources/config/openapi-validator.yml b/light-rest-4j/src/main/resources/handlerconfig/openapi-validator.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/openapi-validator.yml rename to light-rest-4j/src/main/resources/handlerconfig/openapi-validator.yml diff --git a/light-rest-4j/src/main/resources/config/sanitizer.yml b/light-rest-4j/src/main/resources/handlerconfig/sanitizer.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/sanitizer.yml rename to light-rest-4j/src/main/resources/handlerconfig/sanitizer.yml diff --git a/light-rest-4j/src/main/resources/config/traceability.yml b/light-rest-4j/src/main/resources/handlerconfig/traceability.yml similarity index 100% rename from light-rest-4j/src/main/resources/config/traceability.yml rename to light-rest-4j/src/main/resources/handlerconfig/traceability.yml diff --git a/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java b/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java index adc92327f..bc1c014cf 100644 --- a/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java +++ b/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java @@ -140,17 +140,43 @@ public void testInvalidVaribleNameGeneratorJson() throws IOException { } @Test - public void testYAMLFileParameterizer() { - File srcDir = new File("src/main/resources/config/"); - File destDir = new File("/tmp/test/"); + public void testResourceParameterizing() throws IOException { + String destDirName = "/tmp/yml_param_test"; + File destDir = new File(destDirName); + + if (destDir.exists() && destDir.isDirectory()) { + destDir.delete(); + } FilenameFilter filter = (dir, name)->name.toLowerCase().endsWith(".yml"); - YAMLFileParameterizer.rewriteAll(srcDir, destDir, true); + Any anyConfig = JsonIterator.parse(OpenApiGeneratorTest.class.getResourceAsStream(configName), 1024).readAny(); + + YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_RESOURCE_LOCATION, destDirName, anyConfig.get(OpenApiGenerator.GENERATE_ENV_VARS).asMap()); + + int destCount = destDir.list(filter).length; + + assertTrue(destCount>0); + } + + @Test + public void testFileParameterizing() throws IOException { + String srcDirName = "src/main/resources/handlerconfig"; + String destDirName = "/tmp/yml_param_test"; + File destDir = new File(destDirName); + + if (destDir.exists() && destDir.isDirectory()) { + destDir.delete(); + } + + FilenameFilter filter = (dir, name)->name.toLowerCase().endsWith(".yml"); + + Any anyConfig = JsonIterator.parse(OpenApiGeneratorTest.class.getResourceAsStream(configName), 1024).readAny(); + + YAMLFileParameterizer.rewriteAll(srcDirName, destDirName, anyConfig.get(OpenApiGenerator.GENERATE_ENV_VARS).asMap()); - int srcCount = srcDir.list(filter).length; int destCount = destDir.list(filter).length; - assertTrue(srcCount==destCount); + assertTrue(destCount>0); } } diff --git a/light-rest-4j/src/test/resources/config.json b/light-rest-4j/src/test/resources/config.json index 0d34ea488..48c773e82 100644 --- a/light-rest-4j/src/test/resources/config.json +++ b/light-rest-4j/src/test/resources/config.json @@ -24,5 +24,10 @@ }, "supportH2ForTest": false, "supportClient": false, - "dockerOrganization": "networknt" + "dockerOrganization": "networknt", + "generateEnvVars": { + "generate": true, + "skipArray": true, + "skipMap": true + } } From 281d751458f0dc3bbef49c4ff1c79ced3c29d104 Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Wed, 13 Feb 2019 13:16:27 -0500 Subject: [PATCH 06/10] fix formatting issue - #216, #217 --- .../com/networknt/codegen/rest/YAMLFileParameterizer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java index f01470652..92ebeec18 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java @@ -258,13 +258,13 @@ protected static List parameterize(String filename, List srcline ScalarNode sv = (ScalarNode)v; copy(srclines, destlines, pos, startLine); - destlines.add(String.format("%s:${%s%s%s:%s}", sk.getValue(), filename, KEY_SEPARATOR, sk.getValue(), sv.getValue())); + destlines.add(String.format("%s: ${%s%s%s:%s}", sk.getValue(), filename, KEY_SEPARATOR, sk.getValue(), sv.getValue())); pos = stopLine+1; }else { if (!skipArray && !skipMap) { copy(srclines, destlines, pos, startLine); - destlines.add(String.format("%s:${%s%s%s}", sk.getValue(), filename, KEY_SEPARATOR, sk.getValue())); + destlines.add(String.format("%s: ${%s%s%s}", sk.getValue(), filename, KEY_SEPARATOR, sk.getValue())); pos = stopLine+1; }else { From d4a95ef21c90b7c8618911da33a6c83e6b54ce1e Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Wed, 13 Feb 2019 13:21:00 -0500 Subject: [PATCH 07/10] refine filtering condition --- .../com/networknt/codegen/rest/YAMLFileParameterizer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java index 92ebeec18..9c10f3aa6 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java @@ -23,6 +23,7 @@ import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.NodeTuple; import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; import org.yaml.snakeyaml.parser.ParserImpl; import org.yaml.snakeyaml.reader.StreamReader; import org.yaml.snakeyaml.resolver.Resolver; @@ -262,7 +263,8 @@ protected static List parameterize(String filename, List srcline pos = stopLine+1; }else { - if (!skipArray && !skipMap) { + if (((v instanceof SequenceNode) && !skipArray) + || ((v instanceof MappingNode) && !skipMap)) { copy(srclines, destlines, pos, startLine); destlines.add(String.format("%s: ${%s%s%s}", sk.getValue(), filename, KEY_SEPARATOR, sk.getValue())); From baef5520aa42a788f3691c15fd0f48d9b979cf7d Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Wed, 13 Feb 2019 16:51:39 -0500 Subject: [PATCH 08/10] fix defect of read files in jar - #216, #217 --- .../codegen/rest/YAMLFileParameterizer.java | 113 ++++++++++++++---- 1 file changed, 89 insertions(+), 24 deletions(-) diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java index 9c10f3aa6..17f4afe15 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java @@ -8,9 +8,16 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; import java.util.List; import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; @@ -44,6 +51,11 @@ public class YAMLFileParameterizer { protected static final String KEY_SKIP_MAP="skipMap"; public static void rewriteAll(String srcLocation, String destDir, Map generateEnvVars) { + + if (logger.isDebugEnabled()) { + logger.debug("rewriting files in {}", srcLocation); + } + if (fromClasspath(srcLocation)) { rewriteResources(resolveLocation(srcLocation), destDir, generateEnvVars); }else { @@ -73,27 +85,29 @@ public static void rewriteFiles(File sourceDir, File destDir, Map g public static void rewriteResources(String resourceLocation, String destDir, Map generateEnvVars) { - try { - List filenames = IOUtils.readLines(YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation), (String)null); - - List ymlFileNames = filenames.stream().filter(name->name.toLowerCase().endsWith(YML_EXT)).collect(Collectors.toList()); - - File dest = new File(destDir); - - if (!dest.isDirectory() || !dest.exists()) { - if (!dest.mkdir()) { - logger.error("Failed to create dir {}", destDir); - return; - } - } - - for (String filename: ymlFileNames) { - rewriteResource(stripExtension(filename), resourceLocation+File.separator+filename, destDir+File.separator+filename, generateEnvVars); + List filenames = listClasspathDir(resourceLocation); + + if (logger.isDebugEnabled()) { + logger.debug("files in {}: {}", resourceLocation, String.join(",", filenames)); + } + + List ymlFileNames = filenames.stream().filter(name->name.toLowerCase().endsWith(YML_EXT)).collect(Collectors.toList()); + + if (logger.isDebugEnabled()) { + logger.debug("ymlFileNames in {}: {}", resourceLocation, String.join(",", ymlFileNames)); + } + + File dest = new File(destDir); + + if (!dest.isDirectory() || !dest.exists()) { + if (!dest.mkdir()) { + logger.error("Failed to create dir {}", destDir); + return; } - - - } catch (IOException e) { - logger.error(e.getMessage(), e); + } + + for (String filename: ymlFileNames) { + rewriteResource(stripExtension(filename), resourceLocation+filename, destDir+File.separator+filename, generateEnvVars); } } @@ -154,7 +168,7 @@ protected static boolean getValue(Map generateEnvVars, String key) } protected static void copyResource(String resourceLocation, String destFilename) { - try (InputStream in = YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation); + try (InputStream in = getResourceAsStream(resourceLocation); FileOutputStream out = new FileOutputStream(destFilename)) { IOUtils.copy(in, out); } catch (IOException e) { @@ -176,8 +190,58 @@ protected static void copyFile(File srcFile, File destFile) { } } + protected static URL getResourceURL(String resource) { + return YAMLFileParameterizer.class.getClassLoader().getResource(resource); + } + + protected static InputStream getResourceAsStream(String resource) { + return YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resource); + } + + protected static List listClasspathDir(String dir) { + List result = new ArrayList<>(); + + try { + URL dirURL = getResourceURL(dir); + + if (null==dirURL) { + logger.error("cannot file {} in classpath.", dir); + return result; + } + + if (dirURL.getProtocol().equals("file")) { + result.addAll(Arrays.asList(new File(dirURL.toURI()).list())); + return result; + }else if (dirURL.getProtocol().equals("jar")) { /* A JAR path */ + // strip out only the JAR file + String path = dirURL.getPath(); + String jarPath = path.substring("file:".length(), path.indexOf("!")); + + JarFile jar = new JarFile(URLDecoder.decode(jarPath, "UTF-8")); + + Enumeration entries = jar.entries(); // gives ALL entries in jar + int length = dir.length(); + + while (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + + int pathIndex = name.lastIndexOf(dir); + if (pathIndex >= 0) { + result.add(name.substring(pathIndex+length)); + } + } + jar.close(); + return result; + } + } catch(Exception e) { + logger.error(e.getMessage(), e); + } + + throw new UnsupportedOperationException("Cannot list files in " + dir); + } + protected static Node loadResource(String resourceLocation) { - try (InputStream in = YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation); + try (InputStream in = getResourceAsStream(resourceLocation); BufferedReader inputReader = new BufferedReader(new InputStreamReader(in))) { Composer composer = new Composer(new ParserImpl(new StreamReader(inputReader)), new Resolver()); return composer.getSingleNode(); @@ -189,8 +253,9 @@ protected static Node loadResource(String resourceLocation) { } protected static List readResource(String resourceLocation) { - try (InputStream in = YAMLFileParameterizer.class.getClassLoader().getResourceAsStream(resourceLocation)) { - return IOUtils.readLines(in, (String) null); + try (InputStream in = getResourceAsStream(resourceLocation); + InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) { + return IOUtils.readLines(reader); } catch (Exception e) { logger.error(e.getMessage(), e); } From b21879c30ef790e7a16476ef52e0ecf679139a5d Mon Sep 17 00:00:00 2001 From: Daniel Zhao Date: Thu, 14 Feb 2019 13:58:02 -0500 Subject: [PATCH 09/10] add 'exclude' config and rewrite all config files - #216, #217 --- .../codegen/rest/OpenApiGenerator.java | 9 +- .../codegen/rest/YAMLFileParameterizer.java | 263 ++++++++++++++++-- .../handlerconfig/openapi-validator.yml | 6 - .../com/networknt/codegen/ConfigTest.java | 2 + .../codegen/OpenApiGeneratorTest.java | 45 --- .../rest/YAMLFileParameterizerTest.java | 84 ++++++ light-rest-4j/src/test/resources/config.json | 6 +- 7 files changed, 329 insertions(+), 86 deletions(-) delete mode 100644 light-rest-4j/src/main/resources/handlerconfig/openapi-validator.yml create mode 100644 light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java index 97320d60b..3e35b55a5 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/OpenApiGenerator.java @@ -51,8 +51,6 @@ public class OpenApiGenerator implements Generator { private Map typeMapping = new HashMap<>(); - public static final String GENERATE_ENV_VARS="generateEnvVars"; - // optional generation parameters. if not set, they use default values as boolean prometheusMetrics =false; boolean skipHealthCheck = false; @@ -189,8 +187,11 @@ public void generate(String targetPath, Object model, Any config) throws IOExcep if(generateValuesYml) transfer(targetPath, ("src.main.resources.config").replace(".", separator), "values.yml", templates.rest.openapi.values.template()); - if (config.keys().contains(GENERATE_ENV_VARS)) { - YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_RESOURCE_LOCATION, targetPath+separator+YAMLFileParameterizer.DEFAULT_DEST_DIR, config.get(GENERATE_ENV_VARS).asMap()); + //always copy resources + YAMLFileParameterizer.copyResources(YAMLFileParameterizer.DEFAULT_RESOURCE_LOCATION, targetPath+separator+YAMLFileParameterizer.DEFAULT_DEST_DIR); + + if (config.keys().contains(YAMLFileParameterizer.GENERATE_ENV_VARS)) { + YAMLFileParameterizer.rewriteAll(targetPath+separator+YAMLFileParameterizer.DEFAULT_DEST_DIR, config.get(YAMLFileParameterizer.GENERATE_ENV_VARS).asMap()); } } } diff --git a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java index 17f4afe15..6d469e1df 100644 --- a/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java +++ b/light-rest-4j/src/main/java/com/networknt/codegen/rest/YAMLFileParameterizer.java @@ -13,9 +13,12 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.stream.Collectors; @@ -41,7 +44,10 @@ public class YAMLFileParameterizer { private static final Logger logger = LoggerFactory.getLogger(YAMLFileParameterizer.class); private static final String KEY_SEPARATOR = "."; private static final String YML_EXT = ".yml"; + private static final String BAK_EXT = ".bak"; + private static final String SLASH = "/"; + public static final String GENERATE_ENV_VARS="generateEnvVars"; public static final String CLASS_PATH_PREFIX="classpath:"; public static final String DEFAULT_RESOURCE_LOCATION = CLASS_PATH_PREFIX+"handlerconfig/"; public static final String DEFAULT_DEST_DIR = "src/main/resources/config"; @@ -49,6 +55,23 @@ public class YAMLFileParameterizer { protected static final String KEY_GENERATE="generate"; protected static final String KEY_SKIP_ARRAY="skipArray"; protected static final String KEY_SKIP_MAP="skipMap"; + protected static final String KEY_IN_PLACE="inPlace"; + protected static final String KEY_EXCLUDE="exclude"; + + /** + * In place rewriting. + * + * @param dir - file dir + * @param generateEnvVars - config + */ + public static void rewriteAll(String dir, Map generateEnvVars) { + if (logger.isDebugEnabled()) { + logger.debug("rewriting files in {}", dir); + } + + generateEnvVars.put(KEY_IN_PLACE, Any.wrap(true)); + rewriteFiles(new File(dir), new File(dir), generateEnvVars); + } public static void rewriteAll(String srcLocation, String destDir, Map generateEnvVars) { @@ -70,28 +93,49 @@ public static void rewriteFiles(File sourceDir, File destDir, Map g } if (!destDir.isDirectory() || !destDir.exists()) { - if (!destDir.mkdir()) { + if (!destDir.mkdirs()) { logger.error("Failed to create dir {}", destDir); return; } } + + String dirPath = getAbsolutePath(sourceDir); + + if (null==dirPath) { + logger.error("dir does not exist.", sourceDir.getAbsolutePath()); + return; + } + + Set excludeSet = buildFileExcludeSet(dirPath, generateEnvVars); - File[] files = sourceDir.listFiles((dir, name)->name.toLowerCase().endsWith(YML_EXT)); + File[] files = sourceDir.listFiles((dir, name)->name.toLowerCase().endsWith(YML_EXT) && !excludeFile(dir, name, excludeSet)); for (File file: files) { rewriteFile(file, new File(destDir, file.getName()), generateEnvVars); } } - public static void rewriteResources(String resourceLocation, String destDir, Map generateEnvVars) { + if (StringUtils.isBlank(resourceLocation)) { + return; + } + + String location = StringUtils.trimToEmpty(resourceLocation); + if (!location.endsWith(SLASH)) { + location = location+SLASH; + } + List filenames = listClasspathDir(resourceLocation); if (logger.isDebugEnabled()) { logger.debug("files in {}: {}", resourceLocation, String.join(",", filenames)); } - List ymlFileNames = filenames.stream().filter(name->name.toLowerCase().endsWith(YML_EXT)).collect(Collectors.toList()); + Set excludeSet = buildResourceExcludeSet(resourceLocation, generateEnvVars); + + List ymlFileNames = filenames.stream() + .filter(name->name.toLowerCase().endsWith(YML_EXT) && !excludeResource(resourceLocation, name, excludeSet)) + .collect(Collectors.toList()); if (logger.isDebugEnabled()) { logger.debug("ymlFileNames in {}: {}", resourceLocation, String.join(",", ymlFileNames)); @@ -100,7 +144,7 @@ public static void rewriteResources(String resourceLocation, String destDir, Map File dest = new File(destDir); if (!dest.isDirectory() || !dest.exists()) { - if (!dest.mkdir()) { + if (!dest.mkdirs()) { logger.error("Failed to create dir {}", destDir); return; } @@ -111,6 +155,35 @@ public static void rewriteResources(String resourceLocation, String destDir, Map } } + public static void copyResources(String resourceLocation, String destDir) { + String location= resolveLocation(resourceLocation); + + List filenames = listClasspathDir(location); + + if (logger.isDebugEnabled()) { + logger.debug("files in {}: {}", resourceLocation, String.join(",", filenames)); + } + + List ymlFileNames = filenames.stream().filter(name->name.toLowerCase().endsWith(YML_EXT)).collect(Collectors.toList()); + + if (logger.isDebugEnabled()) { + logger.debug("ymlFileNames in {}: {}", resourceLocation, String.join(",", ymlFileNames)); + } + + File dest = new File(destDir); + + if (!dest.isDirectory() || !dest.exists()) { + if (!dest.mkdirs()) { + logger.error("Failed to create dir {}", destDir); + return; + } + } + + for (String filename: ymlFileNames) { + copyResource(location+filename, destDir+File.separator+filename); + } + } + public static void rewriteResource(String filename, String resourceLocation, String destFilename, Map generateEnvVars) { if (logger.isDebugEnabled()) { logger.debug("rewriting resource {}", resourceLocation); @@ -134,15 +207,47 @@ public static void rewriteFile(File srcFile, File destFile, Map gen } if (!getValue(generateEnvVars, KEY_GENERATE)) { - copyFile(srcFile, destFile); + if (!getValue(generateEnvVars, KEY_IN_PLACE)) {// in-place copy is meaningless + copyFile(srcFile, destFile); + } }else { - Node document = loadFile(srcFile); - List lines = readFile(srcFile); - - String filename = stripExtension(srcFile.getName()); - - if (null!=document && null!=lines) { - parameterize(filename, document, lines, destFile, generateEnvVars); + try { + String filename = stripExtension(srcFile.getName()); + + String srcFilename = getAbsolutePath(srcFile); + + if (null==srcFilename) { + logger.error("file {} does not exist.", srcFile.getAbsolutePath()); + return; + } + + File inputFile = srcFile; + boolean inPlace = getValue(generateEnvVars, KEY_IN_PLACE); + + if (inPlace) {// rename the original file as a back up + String bakFilename = srcFilename + BAK_EXT; + File bakFile = new File(bakFilename); + + if (!srcFile.renameTo(bakFile)) { + logger.error("Failed to reanme file {} to {}.", srcFilename, bakFilename); + return; + } + + inputFile = bakFile; + } + + List lines = readFile(inputFile); + Node document = loadFile(inputFile); + + if (null!=document && null!=lines) { + parameterize(filename, document, lines, destFile, generateEnvVars); + } + + if (inPlace) {// delete the original file as it's not needed any more. + inputFile.delete(); + } + }catch(Exception e) { + logger.error(e.getMessage(), e); } } } @@ -205,7 +310,7 @@ protected static List listClasspathDir(String dir) { URL dirURL = getResourceURL(dir); if (null==dirURL) { - logger.error("cannot file {} in classpath.", dir); + logger.error("cannot locate file {} in classpath.", dir); return result; } @@ -253,14 +358,17 @@ protected static Node loadResource(String resourceLocation) { } protected static List readResource(String resourceLocation) { + List lines = new ArrayList<>(); + try (InputStream in = getResourceAsStream(resourceLocation); InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) { - return IOUtils.readLines(reader); + lines.addAll(IOUtils.readLines(reader)); + lines.add(StringUtils.EMPTY); // because trailing empty lines are ignored. } catch (Exception e) { logger.error(e.getMessage(), e); } - return null; + return lines; } protected static Node loadFile(File file) { @@ -275,13 +383,16 @@ protected static Node loadFile(File file) { } protected static List readFile(File file) { + List lines = new ArrayList<>(); + try { - return FileUtils.readLines(file, (String) null); + lines.addAll(FileUtils.readLines(file, (String) null)); + lines.add(StringUtils.EMPTY); // because trailing empty lines are ignored. } catch (Exception e) { logger.error(e.getMessage(), e); } - return null; + return lines; } protected static void parameterize(String filename, Node document, List srclines, File destFile, Map generateEnvVars) { @@ -310,38 +421,50 @@ protected static List parameterize(String filename, List srcline int pos = 0; - for (NodeTuple tuple: tuples) { + int tupleNum = tuples.size(); + + for (int i=0; i parameterize(String filename, List srcline } protected static void copy(List srclines, List destlines, int start, int end) { - if (end>srclines.size()) { + if (end>=srclines.size()) { logger.error("attemp to read line {}. total length {}", end, srclines.size()); throw new IndexOutOfBoundsException(); } - for (int i=start; i buildFileExcludeSet(String sourceDir, Map generateEnvVars) { + if (generateEnvVars.containsKey(KEY_EXCLUDE)) { + return buildFileExcludeSet(sourceDir, generateEnvVars.get(KEY_EXCLUDE).asList()); + } + + return Collections.emptySet(); + } + + protected static Set buildFileExcludeSet(String sourceDir, Collection excludes) { + return excludes.stream() + .map(Any::toString) + .filter(StringUtils::isNotBlank) + .map(s->sourceDir+normalizeFilename(s)) + .collect(Collectors.toSet()); + } + + protected static Set buildResourceExcludeSet(String resourceLocation, Map generateEnvVars) { + if (generateEnvVars.containsKey(KEY_EXCLUDE)) { + return buildResourceExcludeSet(resourceLocation, generateEnvVars.get(KEY_EXCLUDE).asList()); + } + + return Collections.emptySet(); + } + + protected static Set buildResourceExcludeSet(String resourceLocation, Collection excludes) { + return excludes.stream() + .map(Any::toString) + .filter(StringUtils::isNotBlank) + .map(s->toNonNullString(getResourceURL(resourceLocation+s))) + .collect(Collectors.toSet()); + } + + protected static String normalizeFilename(String filename) { + return StringUtils.trimToEmpty(filename).replaceAll("\\\\|/", File.separator); + } + + public static String toNonNullString(URL url) { + return null==url?StringUtils.EMPTY:url.toString(); + } + + public static String getAbsolutePath(File f) { + try { + String path = f.getCanonicalFile().getAbsolutePath(); + + if (f.isDirectory() && !path.endsWith(File.separator)) { + path = path+File.separator; + } + + return path; + }catch (Exception e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + protected static boolean excludeFile(File f, Set excludeSet) { + if (excludeSet.isEmpty()) { + return false; + } + + String filename = getAbsolutePath(f); + + return null == filename ? true : excludeSet.contains(filename); + } + + protected static boolean excludeFile(File dir, String name, Set excludeSet) { + return excludeFile(new File(dir, name), excludeSet); + } + + protected static boolean excludeResource(String resourceLocation, String name, Set excludeSet) { + if (excludeSet.isEmpty()) { + return false; + } + + String url = toNonNullString(getResourceURL(resourceLocation+name)); + + return excludeSet.contains(url); + } @SuppressWarnings("serial") public static class UnsupportedNodeTypeException extends RuntimeException{ diff --git a/light-rest-4j/src/main/resources/handlerconfig/openapi-validator.yml b/light-rest-4j/src/main/resources/handlerconfig/openapi-validator.yml deleted file mode 100644 index 717bd7e83..000000000 --- a/light-rest-4j/src/main/resources/handlerconfig/openapi-validator.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Default openapi-validator configuration ---- -# Enable request validation. Response validation is not done on the server but client. -enabled: true -# Log error message if validation error occurs -logError: true diff --git a/light-rest-4j/src/test/java/com/networknt/codegen/ConfigTest.java b/light-rest-4j/src/test/java/com/networknt/codegen/ConfigTest.java index bc3213f24..bf63427f7 100644 --- a/light-rest-4j/src/test/java/com/networknt/codegen/ConfigTest.java +++ b/light-rest-4j/src/test/java/com/networknt/codegen/ConfigTest.java @@ -27,4 +27,6 @@ public void testDbName() { public void testDbSupport() { Assert.assertTrue(anyConfig.toBoolean("supportDb")); } + + } diff --git a/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java b/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java index bc1c014cf..ec7a1c0b6 100644 --- a/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java +++ b/light-rest-4j/src/test/java/com/networknt/codegen/OpenApiGeneratorTest.java @@ -1,9 +1,6 @@ package com.networknt.codegen; -import static org.junit.Assert.assertTrue; - import java.io.File; -import java.io.FilenameFilter; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; @@ -18,7 +15,6 @@ import com.jsoniter.JsonIterator; import com.jsoniter.any.Any; import com.networknt.codegen.rest.OpenApiGenerator; -import com.networknt.codegen.rest.YAMLFileParameterizer; import com.thoughtworks.qdox.JavaProjectBuilder; import com.thoughtworks.qdox.model.JavaClass; import com.thoughtworks.qdox.model.JavaField; @@ -138,45 +134,4 @@ public void testInvalidVaribleNameGeneratorJson() throws IOException { } } } - - @Test - public void testResourceParameterizing() throws IOException { - String destDirName = "/tmp/yml_param_test"; - File destDir = new File(destDirName); - - if (destDir.exists() && destDir.isDirectory()) { - destDir.delete(); - } - - FilenameFilter filter = (dir, name)->name.toLowerCase().endsWith(".yml"); - - Any anyConfig = JsonIterator.parse(OpenApiGeneratorTest.class.getResourceAsStream(configName), 1024).readAny(); - - YAMLFileParameterizer.rewriteAll(YAMLFileParameterizer.DEFAULT_RESOURCE_LOCATION, destDirName, anyConfig.get(OpenApiGenerator.GENERATE_ENV_VARS).asMap()); - - int destCount = destDir.list(filter).length; - - assertTrue(destCount>0); - } - - @Test - public void testFileParameterizing() throws IOException { - String srcDirName = "src/main/resources/handlerconfig"; - String destDirName = "/tmp/yml_param_test"; - File destDir = new File(destDirName); - - if (destDir.exists() && destDir.isDirectory()) { - destDir.delete(); - } - - FilenameFilter filter = (dir, name)->name.toLowerCase().endsWith(".yml"); - - Any anyConfig = JsonIterator.parse(OpenApiGeneratorTest.class.getResourceAsStream(configName), 1024).readAny(); - - YAMLFileParameterizer.rewriteAll(srcDirName, destDirName, anyConfig.get(OpenApiGenerator.GENERATE_ENV_VARS).asMap()); - - int destCount = destDir.list(filter).length; - - assertTrue(destCount>0); - } } diff --git a/light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java b/light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java new file mode 100644 index 000000000..b4959eb60 --- /dev/null +++ b/light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java @@ -0,0 +1,84 @@ +package com.networknt.codegen.rest; + +import static java.io.File.separator; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.Test; + +import com.jsoniter.JsonIterator; +import com.jsoniter.any.Any; +import com.networknt.codegen.OpenApiGeneratorTest; + +public class YAMLFileParameterizerTest { + private static String configName = "/config.json"; + + @Test + public void testNormalizeFilename() { + String s1="/a/b/c\\d.txt"; + String s2="\\a\\b\\c\\d.txt"; + + String expected = String.format("%sa%sb%sc%sd.txt", File.separator, File.separator, File.separator, File.separator); + + String ns1 = YAMLFileParameterizer.normalizeFilename(s1); + String ns2 = YAMLFileParameterizer.normalizeFilename(s2); + + assertTrue(expected.equals(ns1)); + assertTrue(expected.equals(ns2)); + } + + @Test + public void testFileExcludeSet() { + List excludes = Arrays.asList(Any.wrap("/a/b/c\\d.txt"), Any.wrap("2.yml"), Any.wrap("\\a\\b\\c\\d.txt"), Any.wrap("")); + + Set excludeSet = YAMLFileParameterizer.buildFileExcludeSet(".",excludes); + + assertTrue(2==excludeSet.size()); + } + + @Test + public void testResourceExcludeSet() { + List excludes = Arrays.asList(Any.wrap("audit.yml"), Any.wrap("body.yml")); + + Set excludeSet = YAMLFileParameterizer.buildResourceExcludeSet("handlerconfig/", excludes); + + assertTrue(2==excludeSet.size()); + } + + @Test + public void testParameterizing() throws IOException { + String destDirName = "/tmp/yml_param_test"; + File destDir = new File(destDirName); + + if (destDir.exists() && destDir.isDirectory()) { + destDir.delete(); + } + + YAMLFileParameterizer.copyResources(YAMLFileParameterizer.DEFAULT_RESOURCE_LOCATION, destDirName+separator+YAMLFileParameterizer.DEFAULT_DEST_DIR); + + Any anyConfig = JsonIterator.parse(OpenApiGeneratorTest.class.getResourceAsStream(configName), 1024).readAny(); + + Map genConfig = anyConfig.get(YAMLFileParameterizer.GENERATE_ENV_VARS).asMap(); + + if (anyConfig.keys().contains(YAMLFileParameterizer.GENERATE_ENV_VARS)) { + YAMLFileParameterizer.rewriteAll(destDirName+separator+YAMLFileParameterizer.DEFAULT_DEST_DIR, genConfig); + } + + FilenameFilter filter = (dir, name)->name.toLowerCase().endsWith(".yml"); + + int destCount = destDir.list(filter).length; + + assertTrue(destCount>0); + + //genConfig.remove(YAMLFileParameterizer.KEY_IN_PLACE); + + //YAMLFileParameterizer.rewriteFile(new File("src/main/resources/handlerconfig/config.yml"), new File("/tmp/config.yml"), genConfig); + } +} diff --git a/light-rest-4j/src/test/resources/config.json b/light-rest-4j/src/test/resources/config.json index 48c773e82..72df874d4 100644 --- a/light-rest-4j/src/test/resources/config.json +++ b/light-rest-4j/src/test/resources/config.json @@ -28,6 +28,10 @@ "generateEnvVars": { "generate": true, "skipArray": true, - "skipMap": true + "skipMap": true, + "exclude": [ + "handerl.yml", + "values.yml" + ] } } From 83d482077d3344beabe4e5be1413d3ca6d974906 Mon Sep 17 00:00:00 2001 From: Nicholas Azar Date: Fri, 15 Feb 2019 17:24:10 -0500 Subject: [PATCH 10/10] Fix a failing test --- .../com/networknt/codegen/rest/YAMLFileParameterizerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java b/light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java index b4959eb60..c36c19fff 100644 --- a/light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java +++ b/light-rest-4j/src/test/java/com/networknt/codegen/rest/YAMLFileParameterizerTest.java @@ -72,8 +72,8 @@ public void testParameterizing() throws IOException { } FilenameFilter filter = (dir, name)->name.toLowerCase().endsWith(".yml"); - - int destCount = destDir.list(filter).length; + File outputDestDir = new File(destDirName+separator+YAMLFileParameterizer.DEFAULT_DEST_DIR); + int destCount = outputDestDir.list(filter).length; assertTrue(destCount>0);