From 6015ca50407a811591a736ba17febaf1ec254dd9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 30 Jul 2019 18:39:17 +0200 Subject: [PATCH 1/3] Add a general method for executing npm commands that is used by the gui task --- project/BuildUtility.scala | 107 +++++++++++++++---------------------- 1 file changed, 43 insertions(+), 64 deletions(-) diff --git a/project/BuildUtility.scala b/project/BuildUtility.scala index dc408456..2787d9d6 100644 --- a/project/BuildUtility.scala +++ b/project/BuildUtility.scala @@ -139,90 +139,69 @@ class BuildUtility(logger: ManagedLogger) { return } - if (installGuiDeps(guiDir, cacheDir).isEmpty) - return // Early return on failure, error has already been displayed + val packageJson = new File(guiDir, "package.json") - val outDir = buildGui(guiDir, cacheDir) - if (outDir.isEmpty) - return // Again early return on failure - - // Copy built gui into resources, will be included in the classpath on execution of the framework - sbt.IO.copyDirectory(outDir.get, new File("src/main/resources/chatoverflow-gui")) - } - } - - /** - * Download the dependencies of the gui using npm. - * - * @param guiDir the directory of the gui. - * @param cacheDir a dir, where sbt can store files for caching in the "install" sub-dir. - * @return None, if a error occurs which will be displayed, otherwise the output directory with the built gui. - */ - private def installGuiDeps(guiDir: File, cacheDir: File): Option[File] = { - // Check buildGui for a explanation, it's almost the same. - - val install = FileFunction.cached(new File(cacheDir, "install"), FilesInfo.hash)(_ => { - - logger info "Installing GUI dependencies." + if (!executeNpmCommand(guiDir, cacheDir, Set(packageJson), "install", + () => logger error "GUI dependencies couldn't be installed, please check above log for further details.", + () => new File(guiDir, "node_modules") + )) { + return // early return on failure, error has already been displayed + } - val exitCode = new ProcessBuilder(getNpmCommand :+ "install": _*) - .inheritIO() - .directory(guiDir) - .start() - .waitFor() + val srcFiles = recursiveFileListing(new File(guiDir, "src")) + val outDir = new File(guiDir, "dist") - if (exitCode != 0) { - logger error "GUI dependencies couldn't be installed, please check above log for further details." - return None - } else { - logger info "GUI dependencies successfully installed." - Set(new File(guiDir, "node_modules")) + if(!executeNpmCommand(guiDir, cacheDir, srcFiles + packageJson, "run build", + () => logger error "GUI couldn't be built, please check above log for further details.", + () => outDir + )) { + return // again early return on failure } - }) - val input = new File(guiDir, "package.json") - install(Set(input)).headOption + // copy built gui into resources, will be included in the classpath on execution of the framework + sbt.IO.copyDirectory(outDir, new File("src/main/resources/chatoverflow-gui")) + } } /** - * Builds the gui using npm. - * - * @param guiDir the directory of the gui. - * @param cacheDir a dir, where sbt can store files for caching in the "build" sub-dir. - * @return None, if a error occurs which will be displayed, otherwise the output directory with the built gui. - */ - private def buildGui(guiDir: File, cacheDir: File): Option[File] = { + * Executes a npm command in the given directory and skips executing the given command + * if no input files have changed and the output file still exists. + * + * @param workDir the directory in which npm should be executed + * @param cacheDir a directory required for caching using sbt + * @param inputs the input files, which will be used for caching. + * If any one of these files change the cache is invalidated. + * @param command the npm command to execute + * @param failed called if npm returned an non-zero exit code + * @param success called if npm returned successfully. Needs to return a file for caching. + * If the returned file doesn't exist the npm command will ignore the cache. + * @return true if npm returned zero as a exit code and false otherwise + */ + private def executeNpmCommand(workDir: File, cacheDir: File, inputs: Set[File], command: String, + failed: () => Unit, success: () => File): Boolean = { // sbt allows easily to cache our external build using FileFunction.cached // sbt will only invoke the passed function when at least one of the input files (passed in the last line of this method) // has been modified. For the gui these input files are all files in the src directory of the gui and the package.json. // sbt passes these input files to the passed function, but they aren't used, we just instruct npm to build the gui. // sbt invalidates the cache as well if any of the output files (returned by the passed function) doesn't exist anymore. - - val build = FileFunction.cached(new File(cacheDir, "build"), FilesInfo.hash)(_ => { - - logger info "Building GUI." - - val buildExitCode = new ProcessBuilder(getNpmCommand :+ "run" :+ "build": _*) + val cachedFn = FileFunction.cached(new File(cacheDir, command), FilesInfo.hash) { _ => { + val exitCode = new ProcessBuilder(getNpmCommand ++ command.split("\\s+"): _*) .inheritIO() - .directory(guiDir) + .directory(workDir) .start() .waitFor() - if (buildExitCode != 0) { - logger error "GUI couldn't be built, please check above log for further details." - return None + if (exitCode != 0) { + failed() + return false } else { - logger info "GUI successfully built." - Set(new File(guiDir, "dist")) + Set(success()) } - }) - - - val srcDir = new File(guiDir, "src") - val packageJson = new File(guiDir, "package.json") - val inputs = recursiveFileListing(srcDir) + packageJson + } + } - build(inputs).headOption + cachedFn(inputs) + true } private def getNpmCommand: List[String] = { From 2c8c67865469aa6e079aa1770aa2c3667c256465 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 31 Jul 2019 20:09:12 +0200 Subject: [PATCH 2/3] Separate GUI into its own jar and add a servlet to serve the jar's content --- .gitignore | 3 - build.sbt | 8 ++- project/BootstrapUtility.scala | 16 ++--- project/BuildUtility.scala | 37 ++++++++--- project/plugins.sbt | 5 +- src/main/scala/ScalatraBootstrap.scala | 4 +- .../chatoverflow/ui/web/GUIServlet.scala | 62 +++++++++++++++++++ .../chatoverflow/ui/web/Server.scala | 4 +- 8 files changed, 113 insertions(+), 26 deletions(-) create mode 100644 src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala diff --git a/.gitignore b/.gitignore index 76ce2ca6..ea9fcbe0 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,3 @@ project/plugins/project/ # Plugin Data data/ - -# Built gui -/src/main/resources/chatoverflow-gui \ No newline at end of file diff --git a/build.sbt b/build.sbt index b10c28c8..ee3c78f6 100644 --- a/build.sbt +++ b/build.sbt @@ -99,6 +99,13 @@ bs := BootstrapUtility.bootstrapGenTask(streams.value.log, s"$scalaMajorVersion$ deploy := BootstrapUtility.prepareDeploymentTask(streams.value.log, scalaMajorVersion) gui := BuildUtility(streams.value.log).guiTask(guiProjectPath.value, streams.value.cacheDirectory / "gui") +Compile / packageBin := { + BuildUtility(streams.value.log).packageGUITask(guiProjectPath.value, scalaMajorVersion, crossTarget.value) + (Compile / packageBin).value +} + +Compile / unmanagedJars := (crossTarget.value ** "chatoverflow-gui*.jar").classpath + // --------------------------------------------------------------------------------------------------------------------- // UTIL // --------------------------------------------------------------------------------------------------------------------- @@ -117,4 +124,3 @@ lazy val getDependencyList = Def.task[List[ModuleID]] { // Clears the built GUI dirs on clean cleanFiles += baseDirectory.value / guiProjectPath.value / "dist" -cleanFiles += baseDirectory.value / "src" / "main" / "resources" / "chatoverflow-gui" \ No newline at end of file diff --git a/project/BootstrapUtility.scala b/project/BootstrapUtility.scala index d7ebfee8..a162602c 100644 --- a/project/BootstrapUtility.scala +++ b/project/BootstrapUtility.scala @@ -165,19 +165,15 @@ object BootstrapUtility { } /** - * Copies ONE jar file from the source to all target directories. Useful for single packaged jar files. - */ + * Copies all jar files from the source to all target directories. + */ private def copyJars(sourceDirectory: String, targetDirectories: List[String], logger: ManagedLogger): Unit = { val candidates = new File(sourceDirectory) .listFiles().filter(f => f.isFile && f.getName.toLowerCase.endsWith(".jar")) - if (candidates.length != 1) { - logger warn s"Unable to identify jar file in $sourceDirectory" - } else { - for (targetDirectory <- targetDirectories) { - Files.copy(Paths.get(candidates.head.getAbsolutePath), - Paths.get(s"$targetDirectory/${candidates.head.getName}")) - logger info s"Finished copying file '${candidates.head.getAbsolutePath}' to '$targetDirectory'." - } + for (targetDirectory <- targetDirectories; file <- candidates) { + Files.copy(Paths.get(file.getAbsolutePath), + Paths.get(s"$targetDirectory/${file.getName}")) + logger info s"Finished copying file '${file.getAbsolutePath}' to '$targetDirectory'." } } } \ No newline at end of file diff --git a/project/BuildUtility.scala b/project/BuildUtility.scala index 2787d9d6..5b50e56d 100644 --- a/project/BuildUtility.scala +++ b/project/BuildUtility.scala @@ -1,9 +1,13 @@ import java.io.{File, IOException} import java.nio.file.{Files, StandardCopyOption} +import java.util.jar.Manifest +import com.fasterxml.jackson.databind.ObjectMapper import sbt.internal.util.ManagedLogger import sbt.util.{FileFunction, FilesInfo} +import scala.io.Source + /** * A build utility instance handles build tasks and prints debug information using the managed logger. * @@ -151,15 +155,10 @@ class BuildUtility(logger: ManagedLogger) { val srcFiles = recursiveFileListing(new File(guiDir, "src")) val outDir = new File(guiDir, "dist") - if(!executeNpmCommand(guiDir, cacheDir, srcFiles + packageJson, "run build", + executeNpmCommand(guiDir, cacheDir, srcFiles + packageJson, "run build", () => logger error "GUI couldn't be built, please check above log for further details.", () => outDir - )) { - return // again early return on failure - } - - // copy built gui into resources, will be included in the classpath on execution of the framework - sbt.IO.copyDirectory(outDir, new File("src/main/resources/chatoverflow-gui")) + ) } } @@ -212,6 +211,30 @@ class BuildUtility(logger: ManagedLogger) { } } + def packageGUITask(guiProjectPath: String, scalaMajorVersion: String, crossTargetDir: File): Unit = { + val dir = new File(guiProjectPath, "dist") + if (!dir.exists()) { + return + } + + val files = recursiveFileListing(dir) + + // contains tuples with the actual file as the first value and the name with directory in the jar as the second value + val jarEntries = files.map(file => (file, "/chatoverflow-gui/" + dir.toURI.relativize(file.toURI))) + + val guiVersion = getGUIVersion(guiProjectPath) + + sbt.IO.jar(jarEntries, new File(crossTargetDir, s"chatoverflow-gui_$scalaMajorVersion-$guiVersion.jar"), new Manifest()) + } + + private def getGUIVersion(guiProjectPath: String): String = { + val packageJson = Source.fromFile(s"$guiProjectPath/package.json") + val version = new ObjectMapper().reader().readTree(packageJson.mkString).get("version").asText() + + packageJson.close() + version + } + /** * Creates a file listing with all files including files in any sub-dir. * diff --git a/project/plugins.sbt b/project/plugins.sbt index 02d779bd..173234f3 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1,4 @@ -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") \ No newline at end of file +addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") + +// JSON Lib (Jackson) +libraryDependencies += "org.json4s" %% "json4s-jackson" % "3.5.2" \ No newline at end of file diff --git a/src/main/scala/ScalatraBootstrap.scala b/src/main/scala/ScalatraBootstrap.scala index 098487d2..d8d37d81 100644 --- a/src/main/scala/ScalatraBootstrap.scala +++ b/src/main/scala/ScalatraBootstrap.scala @@ -4,7 +4,7 @@ import org.codeoverflow.chatoverflow.ui.web.rest.connector.ConnectorController import org.codeoverflow.chatoverflow.ui.web.rest.events.{EventsController, EventsDispatcher} import org.codeoverflow.chatoverflow.ui.web.rest.plugin.PluginInstanceController import org.codeoverflow.chatoverflow.ui.web.rest.types.TypeController -import org.codeoverflow.chatoverflow.ui.web.{CodeOverflowSwagger, OpenAPIServlet} +import org.codeoverflow.chatoverflow.ui.web.{CodeOverflowSwagger, GUIServlet, OpenAPIServlet} import org.scalatra._ /** @@ -30,5 +30,7 @@ class ScalatraBootstrap extends LifeCycle { context.mount(new PluginInstanceController(), "/instances/*", "instances") context.mount(new ConnectorController(), "/connectors/*", "connectors") context.mount(new OpenAPIServlet(), "/api-docs") + + context.mount(new GUIServlet(), "/*") } } \ No newline at end of file diff --git a/src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala new file mode 100644 index 00000000..5bf3015d --- /dev/null +++ b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala @@ -0,0 +1,62 @@ +package org.codeoverflow.chatoverflow.ui.web + +import java.io.File +import java.net.URI +import java.util.jar.JarFile + +import org.codeoverflow.chatoverflow.WithLogger +import org.eclipse.jetty.http.MimeTypes +import org.eclipse.jetty.util.Loader +import org.scalatra.{ActionResult, ScalatraServlet} + +import scala.io.Source + +/** + * A servlet to serve the GUI files of the chatoverflow-gui dir from the classpath. + * This directory is provided if the gui jar is added on the classpath. + * Responds with an error if the gui jar isn't on the classpath. + */ +class GUIServlet extends ScalatraServlet with WithLogger { + + private val jarFilePath = { + val res = Loader.getResource(s"/chatoverflow-gui/") + + // directory couldn't be found + if (res == null) { + logger error "GUI couldn't be found on the classpath! Has the GUI been built?" + null + } else { + // remove the path inside the jar and only keep the file path to the jar file + val jarPath = res.getFile.split("!").head + logger info s"GUI jar file found at ${new File(".").toURI.relativize(new URI(jarPath))}" + + Some(jarPath) + } + } + + get("/*") { + if (jarFilePath.isEmpty) { + ActionResult(500, "GUI couldn't be found on the classpath! Has the GUI been built?", Map()) + } else { + val jarFile = new JarFile(new File(new URI(jarFilePath.get))) + + val path = if (requestPath == "/") + "/index.html" + else + requestPath + + val entry = jarFile.getEntry(s"/chatoverflow-gui$path") + + val res = if (entry == null) { + ActionResult(404, s"Requested file '$path' couldn't be found in the GUI jar!", Map()) + } else { + contentType = MimeTypes.getDefaultMimeByExtension(entry.getName) + Source.fromInputStream(jarFile.getInputStream(entry)).mkString + } + + response.setHeader("Cache-Control", "no-cache,no-store") + jarFile.close() + res + } + } +} diff --git a/src/main/scala/org/codeoverflow/chatoverflow/ui/web/Server.scala b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/Server.scala index 7e46e95e..7f75c796 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/ui/web/Server.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/Server.scala @@ -1,7 +1,6 @@ package org.codeoverflow.chatoverflow.ui.web import org.codeoverflow.chatoverflow.{ChatOverflow, WithLogger} -import org.eclipse.jetty.util.resource.Resource import org.eclipse.jetty.webapp.WebAppContext import org.scalatra.servlet.ScalatraListener @@ -16,9 +15,8 @@ class Server(val chatOverflow: ChatOverflow, val port: Int) extends WithLogger { private val server = new org.eclipse.jetty.server.Server(port) private val context = new WebAppContext() context.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false") - context.setInitParameter("org.eclipse.jetty.servlet.Default.cacheControl", "no-cache,no-store") context setContextPath "/" - context.setBaseResource(Resource.newClassPathResource("/chatoverflow-gui/")) + context setResourceBase "/" context.addEventListener(new ScalatraListener) server.setHandler(context) From f3bcc6ac2ad7a638c97a22b182f024ee05ec106c Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 3 Aug 2019 15:15:49 +0200 Subject: [PATCH 3/3] Add logs to the gui package task and fix some smaller things --- project/BuildUtility.scala | 27 ++++++++++++++----- project/plugins.sbt | 2 +- .../chatoverflow/ui/web/GUIServlet.scala | 2 +- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/project/BuildUtility.scala b/project/BuildUtility.scala index 5b50e56d..774126f1 100644 --- a/project/BuildUtility.scala +++ b/project/BuildUtility.scala @@ -214,25 +214,38 @@ class BuildUtility(logger: ManagedLogger) { def packageGUITask(guiProjectPath: String, scalaMajorVersion: String, crossTargetDir: File): Unit = { val dir = new File(guiProjectPath, "dist") if (!dir.exists()) { + logger info "GUI hasn't been compiled. Won't create a jar for it." return } val files = recursiveFileListing(dir) // contains tuples with the actual file as the first value and the name with directory in the jar as the second value - val jarEntries = files.map(file => (file, "/chatoverflow-gui/" + dir.toURI.relativize(file.toURI))) + val jarEntries = files.map(file => file -> s"/chatoverflow-gui/${dir.toURI.relativize(file.toURI).toString}") - val guiVersion = getGUIVersion(guiProjectPath) + val guiVersion = getGUIVersion(guiProjectPath).getOrElse("unknown") sbt.IO.jar(jarEntries, new File(crossTargetDir, s"chatoverflow-gui_$scalaMajorVersion-$guiVersion.jar"), new Manifest()) } - private def getGUIVersion(guiProjectPath: String): String = { - val packageJson = Source.fromFile(s"$guiProjectPath/package.json") - val version = new ObjectMapper().reader().readTree(packageJson.mkString).get("version").asText() + private def getGUIVersion(guiProjectPath: String): Option[String] = { + val packageJson = new File(s"$guiProjectPath/package.json") + if (!packageJson.exists()) { + logger error "The package.json file of the GUI doesn't exist. Have you cloned the GUI in the correct directory?" + return None + } + + val content = Source.fromFile(packageJson) + val version = new ObjectMapper().reader().readTree(content.mkString).get("version").asText() + + content.close() - packageJson.close() - version + if (version.isEmpty) { + logger warn "The GUI version couldn't be loaded from the package.json." + None + } else { + Option(version) + } } /** diff --git a/project/plugins.sbt b/project/plugins.sbt index 173234f3..9e16c189 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") -// JSON Lib (Jackson) +// JSON lib (Jackson) used for parsing the GUI version in the package.json file libraryDependencies += "org.json4s" %% "json4s-jackson" % "3.5.2" \ No newline at end of file diff --git a/src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala index 5bf3015d..76687f22 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/GUIServlet.scala @@ -24,7 +24,7 @@ class GUIServlet extends ScalatraServlet with WithLogger { // directory couldn't be found if (res == null) { logger error "GUI couldn't be found on the classpath! Has the GUI been built?" - null + None } else { // remove the path inside the jar and only keep the file path to the jar file val jarPath = res.getFile.split("!").head