From 40268adda27ab2929115e3e2117d43fed499a2ce Mon Sep 17 00:00:00 2001 From: Jose Date: Thu, 19 Jan 2023 08:03:00 -0500 Subject: [PATCH] feat: process lib directory in `.gama` files for ADS projects deployment (#3644) * feat: add processing of lib directory in gama files #3640 * chore: fix sonar bugs #3640 --- docs/admin/developer/agama/lifecycle.md | 6 +- .../src/main/java/io/jans/ads/Deployer.java | 155 ++++++++++++++---- .../io/jans/ads/model/DeploymentDetails.java | 9 + 3 files changed, 137 insertions(+), 33 deletions(-) diff --git a/docs/admin/developer/agama/lifecycle.md b/docs/admin/developer/agama/lifecycle.md index a30f86f3199..ca8191bacb9 100644 --- a/docs/admin/developer/agama/lifecycle.md +++ b/docs/admin/developer/agama/lifecycle.md @@ -173,7 +173,7 @@ You may like to make modifications and enhancements to your flow. There are two https:///jans-config-api/api/v1/agama/source/com.acme.myflow ``` -- Applies a series of modifications to the flow `com.acme.myflow`: nullifies its description, sets the value of configuration properties, and modifies the creation timestamp to *Aug 8th 23:06:40 UTC* +- Applies a series of modifications to the flow `com.acme.myflow`: nullifies its description, sets the value of configuration properties, and modifies the creation timestamp to *Aug 8th 2022 23:06:40 UTC* ``` curl -k -i -H 'Authorization: Bearer ' -H 'Content-Type: application/json-patch+json' @@ -221,7 +221,7 @@ There are two endpoints for retrieval: **Notes:** - Ensure the tokens used have scope `https://jans.io/oauth/config/agama.readonly` -- The response of a successful operation returns a 200 status code with a JSON representation of the flow(s). If some fields result unfamiliar to you, consult the swagger (open api) document linked [above](#creating-a-flow) +- The response of a successful operation returns a 200 status code with a JSON representation of the flow(s). If some fields result unfamiliar to you, consult the swagger (open api) document linked [above](#creating-a-flow) - By default the source code is not included (this may clutter the output considerably). Append `?includeSource=true` to the endpoint URL to have the source in the output Example: @@ -262,4 +262,4 @@ By design Agama is a [transpiled language](./dsl.md#language-compiler) and trans - If this was the first transpilation attempt, i.e. it's a recently created flow, a message like "Source code has errors" will appear in the browser when launching the flow - Otherwise, no error is shown and the flow will behave as if no changes had been applied to the flow's code. This helps preserve the last known "healthy" state of your flow so end-users are not impacted -In any case, the cause of the error can be inspected by [retrieving](#flow-retrieval-and-removal) the flow's data and checking the property `codeError`. \ No newline at end of file +In any case, the cause of the error can be inspected by [retrieving](#flow-retrieval-and-removal) the flow's data and checking the property `codeError`. diff --git a/jans-auth-server/agama/engine/src/main/java/io/jans/ads/Deployer.java b/jans-auth-server/agama/engine/src/main/java/io/jans/ads/Deployer.java index dad03ddebc0..09ef0205147 100644 --- a/jans-auth-server/agama/engine/src/main/java/io/jans/ads/Deployer.java +++ b/jans-auth-server/agama/engine/src/main/java/io/jans/ads/Deployer.java @@ -52,6 +52,7 @@ import org.slf4j.Logger; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; /* * This bean deploys .gama project files. Modifications of this file must not only account a single @@ -61,11 +62,18 @@ public class Deployer { private static final String BASE_DN = "ou=deployments,ou=agama,o=jans"; + private static final String CUST_LIBS_DIR = "/opt/jans/jetty/jans-auth/custom/libs"; private static final String ASSETS_DIR = "/opt/jans/jetty/jans-auth/agama"; + private static final String[] ASSETS_SUBDIRS = { "ftl", "fl" }; + private static final String SCRIPTS_SUBDIR = "scripts"; + private static final String[] TEMPLATES_EXTENSIONS = new String[] { "ftl", "ftlh" }; + private static final String[] SCRIPTS_EXTENSIONS = new String[] { "java", "groovy" }; private static final String FLOW_EXT = "flow"; + private static final String METADATA_FILE = "project.json"; + private static final boolean ON_CONTAINERS = System.getenv("CN_VERSION") != null; @Inject private ObjectMapper mapper; @@ -88,6 +96,7 @@ public class Deployer { private Map projectsFinishTimes; private Map> projectsBasePaths; private Map> projectsFlows; + private Map> projectsLibs; public void process() throws IOException { @@ -125,14 +134,15 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept logger.info("Marking deployment task as active"); //This merge helps other nodes/pods not to take charge of this very deployment task entryManager.merge(dep); - - //Check the zip has the expected layout + Path p = extractGamaFile(b64EncodedAssets); String tmpdir = p.toString(); dd.setProjectMetadata(computeMetadata(name, tmpdir)); + //Check the zip has the expected layout Path pcode = Paths.get(tmpdir, "code"); Path pweb = Paths.get(tmpdir, "web"); + Path plib = Paths.get(tmpdir, "lib"); if (Files.isDirectory(pcode) && Files.isDirectory(pweb)) { @@ -141,8 +151,10 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept if (dd.getError() == null) { projectsFlows.put(prjId, flowIds); - ZipFile zip = compileAssetsArchive(p, pweb); - byte[] bytes = extractZipFileWithPurge(zip, ASSETS_DIR, projectsBasePaths.get(prjId)); + Set libsPaths = transferJarFiles(plib); + ZipFile zip = compileAssetsArchive(p, pweb, plib); + byte[] bytes = extractZipFileWithPurge(zip, ASSETS_DIR, + projectsBasePaths.get(prjId), projectsLibs.get(prjId)); Set basePaths = new HashSet<>(); //Update this project's base paths: use the subdirs of web folder @@ -150,8 +162,13 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept .map(pa -> pa.getFileName().toString()).forEach(basePaths::add); basePaths.remove(pweb.getFileName().toString()); projectsBasePaths.put(prjId, basePaths); - + + //Update this project's libs paths + libsPaths.addAll(computeSourcePaths(plib)); + projectsLibs.put(prjId, libsPaths); + dd.setFolders(new ArrayList<>(basePaths)); + dd.setLibs(new ArrayList<>(libsPaths)); //Update binary in DB - not a gama file anymore! dep.setAssets(new String(b64Encoder.encode(bytes), UTF_8)); } @@ -195,12 +212,14 @@ private Set createFlows(Path dir, DeploymentDetails dd) throws IOExcepti (path, attrs) -> attrs.isRegularFile() && path.getFileName().toString().endsWith("." + FLOW_EXT); logger.info("Looking for .{} files under {}", FLOW_EXT, dir); - Map flowsCode = Files.find​(dir, 2, matcher).collect(Collectors.toMap(p -> p, p -> "")); + Map flowsCode = Files.find​(dir, 3, matcher).collect(Collectors.toMap(p -> p, p -> "")); flowsCode = new HashMap<>(flowsCode); //Make map modifiable Set flowsPaths = flowsCode.keySet(); for (Path p: flowsPaths) { - logger.debug("Reading {}", p); + if (logger.isDebugEnabled()) { + logger.debug("Reading {}", p.getFileName()); + } flowsCode.put(p, Files.readString(p)); } @@ -238,7 +257,8 @@ private Set createFlows(Path dir, DeploymentDetails dd) throws IOExcepti meta.setInputs(tresult.getInputs()); meta.setTimeout(tresult.getTimeout()); meta.setTimestamp(System.currentTimeMillis()); - //No displayname, author or description. No handling of properties either + meta.setAuthor(dd.getProjectMetadata().getAuthor()); + //No displayname or description. No handling of properties either String compiled = tresult.getCode(); fl.setMetadata(meta); @@ -285,22 +305,29 @@ private Set createFlows(Path dir, DeploymentDetails dd) throws IOExcepti } - private ZipFile compileAssetsArchive(Path root, Path webroot) throws IOException { + private ZipFile compileAssetsArchive(Path root, Path webroot, Path lib) throws IOException { String rnd = rndName(); Path agama = Files.createDirectory(Paths.get(root.toString(), rnd)); - logger.debug("Created temp directory {}", agama); + String agamStr = agama.toString(); + logger.debug("Created temp directory"); + + Path ftl = Files.createDirectory(Paths.get(agamStr, "ftl")); + Path fl = Files.createDirectory(Paths.get(agamStr, "fl")); + Path scripts = Files.createDirectory(Paths.get(agamStr, SCRIPTS_SUBDIR)); - Path ftl = Files.createDirectory(Paths.get(agama.toString(), "ftl")); - Path fl = Files.createDirectory(Paths.get(agama.toString(), "fl")); - logger.debug("Copying templates to {}", ftl); Files.walkFileTree(webroot, copyVisitor(webroot, ftl, TEMPLATES_EXTENSIONS, true)); logger.debug("Copying assets to {}", fl); Files.walkFileTree(webroot, copyVisitor(webroot, fl, TEMPLATES_EXTENSIONS, false)); - //Make a zip with ftl and fl folders + if (Files.isDirectory(lib)) { + logger.debug("Copying .java and .groovy sources to {}", scripts); + Files.walkFileTree(lib, copyVisitor(lib, scripts, SCRIPTS_EXTENSIONS, true)); + } + + //Make a zip with scripts, ftl, and fl folders ZipParameters params = new ZipParameters(); params.setCompressionMethod(CompressionMethod.STORE); @@ -310,10 +337,55 @@ private ZipFile compileAssetsArchive(Path root, Path webroot) throws IOException ZipFile newZip = new ZipFile(newZipPath.toFile()); newZip.addFolder(ftl.toFile(), params); newZip.addFolder(fl.toFile(), params); + newZip.addFolder(scripts.toFile(), params); return newZip; } + + private Set computeSourcePaths(Path lib) throws IOException { + + BiPredicate matcher = (path, attrs) -> attrs.isRegularFile() && + Stream.of(SCRIPTS_EXTENSIONS).anyMatch(ext -> path.getFileName().toString().endsWith("." + ext)); + + if (Files.isDirectory(lib)) { + String slib = lib.toString(); + + try (Stream stream = Files.find(lib, 20, matcher)) { + return stream.map(Path::toString) + .map(s -> s.substring(slib.length() + 1)).collect(Collectors.toSet()); + } + } + return Collections.emptySet(); + + } + + private Set transferJarFiles(Path lib) throws IOException { + + Set paths = new HashSet<>(); + //All .jar files found at the top level are moved to the custom libs destination. + //This applies for VM-based installations only + if (!ON_CONTAINERS && Files.isDirectory(lib)) { + BiPredicate matcher = + (path, attrs) -> attrs.isRegularFile() && path.getFileName().toString().endsWith(".jar"); + + List list = null; + try (Stream stream = Files.find(lib, 1, matcher)) { + list = stream.collect(Collectors.toList()); + } + logger.debug("Moving {} jar files to custom libs dir", list.size()); + + for (Path jar : list) { + String fn = jar.getFileName().toString(); + paths.add(fn); + + Files.move(jar, Paths.get(CUST_LIBS_DIR, fn), REPLACE_EXISTING); + logger.debug("{} moved", fn); + } + } + return paths; + + } private void updateFlowsAndAssets(List deployments) { @@ -335,6 +407,8 @@ private void updateFlowsAndAssets(List deployments) { //If local map does not contain the given project or the local finishedAt value is less //than the DB value, extract to disk the assets (including a previous directory purge) + //This conditional can only evaluate truthy in a multinode environment (containers) or + //upon application startup in a VM installation if (finishedAt == null || finishedAt < d.getFinishedAt().getTime()) { //Retrieve associated assets String b64EncodedAssets = entryManager.find(d.getDn(), Deployment.class, @@ -342,7 +416,7 @@ private void updateFlowsAndAssets(List deployments) { try { if (finishedAt != null) { - purge(projectsBasePaths.get(prjId)); + purge(projectsBasePaths.get(prjId), projectsLibs.get(prjId)); } if (b64EncodedAssets != null) { extract(b64EncodedAssets, ASSETS_DIR); @@ -369,7 +443,7 @@ private void updateFlowsAndAssets(List deployments) { try { projectsFinishTimes.remove(prjId); - purge(projectsBasePaths.get(prjId)); + purge(projectsBasePaths.get(prjId), projectsLibs.get(prjId)); } catch(IOException e) { logger.error(e.getMessage()); } @@ -378,20 +452,25 @@ private void updateFlowsAndAssets(List deployments) { } } - Set set; projectsFlows.clear(); projectsBasePaths.clear(); + projectsLibs.clear(); //Refresh maps wrt DB content for (Deployment d : depls) { String prjId = d.getId(); + DeploymentDetails dd = d.getDetails(); - set = Optional.ofNullable(d.getDetails().getFlowsError()).map(Map::keySet) + Set set = Optional.ofNullable(dd.getFlowsError()).map(Map::keySet) .orElse(new HashSet<>()); projectsFlows.put(prjId, set); - set = Optional.ofNullable(d.getDetails().getFolders()).map(HashSet::new) + set = Optional.ofNullable(dd.getFolders()).map(HashSet::new) .orElse(new HashSet<>()); projectsBasePaths.put(prjId, set); + + set = Optional.ofNullable(dd.getLibs()).map(HashSet::new) + .orElse(new HashSet<>()); + projectsLibs.put(prjId, set); } } @@ -439,20 +518,35 @@ private static String dnFromQname(String qname) { // ========== File-system related utilities follow: =========== //Walpurgis - private void purge(Set dirs) throws IOException { + private void purge(Set dirs, Set filesToRemove) throws IOException { - if (dirs == null) return; - - for (String dir : dirs) { - for (String subdir : ASSETS_SUBDIRS) { - Path p = Paths.get(ASSETS_DIR, subdir, dir); + if (dirs != null) { + for (String dir : dirs) { + for (String subdir : ASSETS_SUBDIRS) { + Path p = Paths.get(ASSETS_DIR, subdir, dir); - if (Files.isDirectory(p)) { - logger.info("Flushing folder {}", p); - removeDir(p); + if (Files.isDirectory(p)) { + logger.info("Flushing folder {}", p); + removeDir(p); + } } } } + + if (filesToRemove == null) return; + + for (String f : filesToRemove) { + Path p = null; + + if (f.endsWith(".jar")) { + p = Paths.get(CUST_LIBS_DIR, f); + } else { + p = Paths.get(ASSETS_DIR, SCRIPTS_SUBDIR, f); + } + + logger.debug("Removing file {}", f); + Files.deleteIfExists(p); + } } @@ -487,10 +581,10 @@ private Path extractGamaFile(String b64EncodedContents) throws IOException { } private byte[] extractZipFileWithPurge(ZipFile zip, String destination, - Set dirsPurge) throws IOException { + Set dirsPurge, Set filesToRemove) throws IOException { Path zipPath = zip.getFile().toPath(); - purge(dirsPurge); + purge(dirsPurge, filesToRemove); logger.debug("Extracting contents of {} to {}", zipPath, destination); zip.extractAll(destination); @@ -577,6 +671,7 @@ private void init() { projectsBasePaths = new HashMap<>(); projectsFinishTimes = new HashMap<>(); projectsFlows = new HashMap<>(); + projectsLibs = new HashMap<>(); } diff --git a/jans-auth-server/agama/model/src/main/java/io/jans/ads/model/DeploymentDetails.java b/jans-auth-server/agama/model/src/main/java/io/jans/ads/model/DeploymentDetails.java index 23ccb26507c..6ac88b995e0 100644 --- a/jans-auth-server/agama/model/src/main/java/io/jans/ads/model/DeploymentDetails.java +++ b/jans-auth-server/agama/model/src/main/java/io/jans/ads/model/DeploymentDetails.java @@ -9,6 +9,7 @@ public class DeploymentDetails { private List folders; + private List libs; private Map flowsError; private String error; private ProjectMetadata metadata = new ProjectMetadata(); @@ -29,6 +30,14 @@ public void setFolders(List folders) { this.folders = folders; } + public List getLibs() { + return libs; + } + + public void setLibs(List libs) { + this.libs = libs; + } + public String getError() { return error; }