diff --git a/build.gradle b/build.gradle index e307171..b9651c1 100644 --- a/build.gradle +++ b/build.gradle @@ -370,6 +370,7 @@ dependencies { // Note that the java SDK modules use syslog4j, so we'll need to figure something out // there. I doubt any of the apps actually use the logging code that triggers it though testImplementation('org.syslog4j:syslog4j:0.9.46') + testImplementation('uk.org.webcompere:system-stubs-jupiter:2.1.8') // isolate the sdk generated code dependencies from the standard dependencies diff --git a/src/main/java/us/kbase/sdk/tester/ConfigLoader.java b/src/main/java/us/kbase/sdk/tester/ConfigLoader.java index f282578..c54a750 100644 --- a/src/main/java/us/kbase/sdk/tester/ConfigLoader.java +++ b/src/main/java/us/kbase/sdk/tester/ConfigLoader.java @@ -16,7 +16,6 @@ public class ConfigLoader { private final String authAllowInsecure; private final AuthToken token; private final String endPoint; - private final String jobSrvUrl; private final String wsUrl; private final String shockUrl; private final String handleUrl; @@ -61,11 +60,11 @@ public ConfigLoader(Properties props, boolean testMode, String configPathInfo throw new IllegalStateException("Error: KBase services end-point is not set in " + configPathInfo); } - jobSrvUrl = getConfigUrl(props, "job_service_url", endPoint, "userandjobstate"); wsUrl = getConfigUrl(props, "workspace_url", endPoint, "ws"); shockUrl = getConfigUrl(props, "shock_url", endPoint, "shock-api"); handleUrl = getConfigUrl(props, "handle_url", endPoint, "handle_service"); srvWizUrl = getConfigUrl(props, "srv_wiz_url", endPoint, "service_wizard"); + // not sure if this is still used for the ee2 url or not njswUrl = getConfigUrl(props, "njsw_url", endPoint, "njs_wrapper"); catalogUrl = getConfigUrl(props, "catalog_url", endPoint, "catalog"); secureCfgParams = new TreeMap(); @@ -102,10 +101,6 @@ public String getHandleUrl() { return handleUrl; } - public String getJobSrvUrl() { - return jobSrvUrl; - } - public String getNjswUrl() { return njswUrl; } @@ -127,7 +122,6 @@ public void generateConfigProperties(File configPropsFile) throws Exception { try { pw.println("[global]"); pw.println("kbase_endpoint = " + endPoint); - pw.println("job_service_url = " + jobSrvUrl); pw.println("workspace_url = " + wsUrl); pw.println("shock_url = " + shockUrl); pw.println("handle_url = " + handleUrl); diff --git a/src/main/java/us/kbase/sdk/util/DeployConfigGenerator.java b/src/main/java/us/kbase/sdk/util/DeployConfigGenerator.java new file mode 100644 index 0000000..13c1624 --- /dev/null +++ b/src/main/java/us/kbase/sdk/util/DeployConfigGenerator.java @@ -0,0 +1,120 @@ +package us.kbase.sdk.util; + +import static java.util.Objects.requireNonNull; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.ini4j.Ini; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Generates a deploy.cfg file for a KBase service or app */ +public class DeployConfigGenerator { + + private static final Pattern MUSTACHE_ENTRY = Pattern.compile( + "\\{\\{\\s*([a-zA-Z0-9_]+)\\s*\\}\\}" + ); + + /** Generate a KBase service / app deploy.cfg file given a Mustache template and a properties + * ini file. + * The template file will be backed up and then overwritten in place. + * @param templatePath - the path to the template file. + * @param propertiesPath - the path to the properties / .ini file containing the properties + * to be inserted into the mustache template. The properties must be in a section called + * "global". If not provided or the file doesn't exist, a KBASE_ENDPOINT environment variable + * must exist with the kbase endpoint url as the value, which will be used to build the + * properties. + * @throws IOException if file reads or writes fail. + */ + public static void generateDeployConfig(final Path templatePath, final Path propertiesPath) + throws IOException { + requireNonNull(templatePath, "templatePath"); + + final String templateText = Files.readString(templatePath, StandardCharsets.UTF_8); + + // this is recapitulating the behavior or the original prepare_deploy_cfg.py code + // for now. We might want to be more stringent about the input vs. silently accepting + // whatever. + final Map props; + if (propertiesPath != null && Files.exists(propertiesPath)) { + props = new HashMap<>(); + final Ini ini = new Ini(propertiesPath.toFile()); + if (ini.get("global") != null) { + ini.get("global").forEach(props::put); + } + } else if (System.getenv("KBASE_ENDPOINT") != null) { + props = loadFromEnv(); + } else { + if (propertiesPath == null) { + throw new IllegalArgumentException( + "Properties file was not provided and KBASE_ENDPOINT environment " + + "variable not found" + ); + } else { + throw new IllegalArgumentException(String.format( + "Neither %s file nor KBASE_ENDPOINT environment variable found", + propertiesPath + )); + } + } + final String output = renderTemplate(templateText, props); + + // Create human-readable backup filename + final String timestamp = LocalDateTime.now() + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH-mm-ss-SSS")); + final Path backupPath = templatePath.resolveSibling( + templatePath.getFileName() + ".bak." + timestamp + ); + + Files.writeString(backupPath, templateText, StandardCharsets.UTF_8); + Files.writeString(templatePath, output, StandardCharsets.UTF_8); + } + + private static Map loadFromEnv() { + final Map props = new HashMap<>(); + final String kbaseEndpoint = System.getenv("KBASE_ENDPOINT"); + + props.put("kbase_endpoint", kbaseEndpoint); + props.put("workspace_url", kbaseEndpoint + "/ws"); + props.put("shock_url", kbaseEndpoint + "/shock-api"); + props.put("handle_url", kbaseEndpoint + "/handle_service"); + props.put("srv_wiz_url", kbaseEndpoint + "/service_wizard"); + props.put("njsw_url", kbaseEndpoint + "/njs_wrapper"); + + final String authServiceUrl = System.getenv("AUTH_SERVICE_URL"); + if (authServiceUrl != null) { + props.put("auth_service_url", authServiceUrl); + } + + final String insecure = System.getenv("AUTH_SERVICE_URL_ALLOW_INSECURE"); + props.put("auth_service_url_allow_insecure", insecure == null ? "false" : insecure); + + System.getenv().forEach((key, value) -> { + if (key.startsWith("KBASE_SECURE_CONFIG_PARAM_")) { + String paramName = key.substring("KBASE_SECURE_CONFIG_PARAM_".length()); + props.put(paramName, value); + } + }); + + return props; + } + + private static String renderTemplate(final String template, final Map props) { + final Matcher matcher = MUSTACHE_ENTRY.matcher(template); + final StringBuffer sb = new StringBuffer(); + while (matcher.find()) { + final String key = matcher.group(1); + final String value = props.getOrDefault(key, ""); + matcher.appendReplacement(sb, Matcher.quoteReplacement(value)); + } + matcher.appendTail(sb); + return sb.toString(); + } +} diff --git a/src/main/resources/us/kbase/sdk/templates/module_deploy_cfg.vm.properties b/src/main/resources/us/kbase/sdk/templates/module_deploy_cfg.vm.properties index 90fa8be..c29da4c 100644 --- a/src/main/resources/us/kbase/sdk/templates/module_deploy_cfg.vm.properties +++ b/src/main/resources/us/kbase/sdk/templates/module_deploy_cfg.vm.properties @@ -1,6 +1,5 @@ [${module_name}] kbase-endpoint = {{ kbase_endpoint }} -job-service-url = {{ job_service_url }} workspace-url = {{ workspace_url }} shock-url = {{ shock_url }} handle-service-url = {{ handle_url }} diff --git a/src/main/resources/us/kbase/sdk/templates/module_test_cfg.vm.properties b/src/main/resources/us/kbase/sdk/templates/module_test_cfg.vm.properties index d50f6c8..dab6de4 100644 --- a/src/main/resources/us/kbase/sdk/templates/module_test_cfg.vm.properties +++ b/src/main/resources/us/kbase/sdk/templates/module_test_cfg.vm.properties @@ -9,7 +9,6 @@ kbase_endpoint=https://appdev.kbase.us/services # Next set of URLs correspond to core services. By default they # are defined automatically based on 'kbase_endpoint': -#job_service_url= #workspace_url= #shock_url= #handle_url= diff --git a/src/test/java/us/kbase/test/sdk/common/TestLocalManagerTest.java b/src/test/java/us/kbase/test/sdk/common/TestLocalManagerTest.java index f089cc7..d41598d 100644 --- a/src/test/java/us/kbase/test/sdk/common/TestLocalManagerTest.java +++ b/src/test/java/us/kbase/test/sdk/common/TestLocalManagerTest.java @@ -57,7 +57,6 @@ public class TestLocalManagerTest { # Next set of URLs correspond to core services. By default they # are defined automatically based on 'kbase_endpoint': - #job_service_url= #workspace_url= #shock_url= #handle_url= diff --git a/src/test/java/us/kbase/test/sdk/util/DeployConfigGeneratorTest.java b/src/test/java/us/kbase/test/sdk/util/DeployConfigGeneratorTest.java new file mode 100644 index 0000000..db0898b --- /dev/null +++ b/src/test/java/us/kbase/test/sdk/util/DeployConfigGeneratorTest.java @@ -0,0 +1,208 @@ +package us.kbase.test.sdk.util; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +import us.kbase.sdk.util.DeployConfigGenerator; + +@ExtendWith(SystemStubsExtension.class) +public class DeployConfigGeneratorTest { + + @TempDir + Path tempDir; + + private void assertBackupExists() throws IOException { + final boolean backupExists = Files.list(tempDir) + .anyMatch(p -> p.getFileName().toString().startsWith("template.cfg.bak.")); + assertThat("Backup file should exist with timestamped name", backupExists, is(true)); + } + + @Test + public void testGenerateDeployConfig() throws Exception { + final Path templateFile = tempDir.resolve("template.cfg"); + final Path propertiesFile = tempDir.resolve("props.ini"); + + final String templateContent = """ + endpoint={{ kbase_endpoint }} + job={{ job_service_url }} + foo=bar + """; + Files.writeString(templateFile, templateContent); + final String props = """ + [global] + kbase_endpoint=https://example.org + job_service_url=https://example.org/job + """; + Files.writeString(propertiesFile, props); + + DeployConfigGenerator.generateDeployConfig(templateFile, propertiesFile); + + final String rendered = Files.readString(templateFile); + final String expected = """ + endpoint=https://example.org + job=https://example.org/job + foo=bar + """; + assertThat("Incorrect template render", rendered, is(expected)); + + assertBackupExists(); + } + + @Test + public void testGenerateDeployConfigNoGlobalSection() throws Exception { + final Path templateFile = tempDir.resolve("template.cfg"); + final Path propertiesFile = tempDir.resolve("props.ini"); + + final String templateContent = "endpoint={{ kbase_endpoint }}\njob={{ job_service_url }}"; + Files.writeString(templateFile, templateContent); + + final String props = """ + [global_fake] + kbase_endpoint=https://example.org + job_service_url=https://example.org/job + """; + Files.writeString(propertiesFile, props); + + DeployConfigGenerator.generateDeployConfig(templateFile, propertiesFile); + + final String rendered = Files.readString(templateFile); + final String expected = """ + endpoint= + job="""; + assertThat("Incorrect template render", rendered, is(expected)); + + assertBackupExists(); + } + + @Test + public void testGenerateDeployConfigNoMustache() throws Exception { + final Path templateFile = tempDir.resolve("template.cfg"); + final Path propertiesFile = tempDir.resolve("props.ini"); + + final String templateContent = "endpoint=hardcoded\njob=also_hardcoded"; + Files.writeString(templateFile, templateContent); + Files.writeString(propertiesFile, ""); + + DeployConfigGenerator.generateDeployConfig(templateFile, propertiesFile); + + final String rendered = Files.readString(templateFile); + final String expected = """ + endpoint=hardcoded + job=also_hardcoded"""; + assertThat("Incorrect template render", rendered, is(expected)); + + assertBackupExists(); + } + + private static String ENV_TEMPLATE = """ + endpoint={{ kbase_endpoint }} + ws={{ workspace_url }} + auth={{ auth_service_url }} + secure={{ auth_service_url_allow_insecure }} + othervar={{ othervar }} + secure1={{ one }} + secure2={{ two }} + foo=bar + """; + + @Test + public void testGenerateDeployConfigEnvVarsMinimal() throws Exception { + final Path templateFile = tempDir.resolve("template.cfg"); + + Files.writeString(templateFile, ENV_TEMPLATE); + + new EnvironmentVariables().set("KBASE_ENDPOINT", "https://ci.kbase.us/services") + // test w/ null props file + .execute(() -> {DeployConfigGenerator.generateDeployConfig(templateFile, null);}); + + String rendered = Files.readString(templateFile); + final String expected = """ + endpoint=https://ci.kbase.us/services + ws=https://ci.kbase.us/services/ws + auth= + secure=false + othervar= + secure1= + secure2= + foo=bar + """; + assertThat("Incorrect template render", rendered, is(expected)); + + assertBackupExists(); + } + + @Test + public void testGenerateDeployConfigEnvVarsFull() throws Exception { + final Path templateFile = tempDir.resolve("template.cfg"); + final Path propsFile = tempDir.resolve("config.props"); + + Files.writeString(templateFile, ENV_TEMPLATE); + + new EnvironmentVariables() + .set("KBASE_ENDPOINT", "https://ci.kbase.us/services") + .set("AUTH_SERVICE_URL", "https://kbase_auth.us") + .set("AUTH_SERVICE_URL_ALLOW_INSECURE", "totes mcgoats") + .set("OTHER_VAR", "shouldn't appear") + .set("KBASE_SECURE_CONFIG_PARAM_one", "super_secure1") + .set("KBASE_SECURE_CONFIG_PARAM_two", "super_secure2") + // test w/ missing props file + .execute(() -> {DeployConfigGenerator.generateDeployConfig(templateFile, propsFile);}); + + final String rendered = Files.readString(templateFile); + final String expected = """ + endpoint=https://ci.kbase.us/services + ws=https://ci.kbase.us/services/ws + auth=https://kbase_auth.us + secure=totes mcgoats + othervar= + secure1=super_secure1 + secure2=super_secure2 + foo=bar + """; + assertThat("Incorrect template render", rendered, is(expected)); + + assertBackupExists(); + } + + @Test + public void testGenerateDeployConfigFail() throws Exception { + final Path t = tempDir.resolve("template.cfg"); + final Path p = tempDir.resolve("config.properties"); + + failGenerateDeployConfig(null, null, new NullPointerException("templatePath")); + failGenerateDeployConfig(t, null, new NoSuchFileException(tempDir + "/template.cfg")); + + Files.writeString(t, "foo"); + + failGenerateDeployConfig(t, null, new IllegalArgumentException( + "Properties file was not provided and KBASE_ENDPOINT environment variable " + + "not found" + )); + failGenerateDeployConfig(t, p, new IllegalArgumentException(String.format( + "Neither %s/config.properties file nor KBASE_ENDPOINT environment variable found", + tempDir + ))); + } + + private void failGenerateDeployConfig( + final Path template, final Path props, final Exception expected) + throws Exception { + final Exception e = assertThrows(expected.getClass(), + () -> DeployConfigGenerator.generateDeployConfig(template, props) + ); + assertThat(e.getMessage(), is(expected.getMessage())); + } + +}