Much better integration of HTMLOptimizer and added support to Templat…

…eCompiler to support HTMLOptimizer integration.
outr · Mar 29, 2017 · bc135c6c9ef11f374f3fe6fdf5fbdb68af77c123 · bc135c6
1 parent 3454da3
commit bc135c6c9ef11f374f3fe6fdf5fbdb68af77c123
Unified Split

Showing with 99 additions and 26 deletions.

+78 −16 optimizer/src/main/scala/io/youi/optimizer/HTMLOptimizer.scala

+21 −10 template/jvm/src/main/scala/io/youi/template/TemplateCompiler.scala
diff --git a/optimizer/src/main/scala/io/youi/optimizer/HTMLOptimizer.scala b/optimizer/src/main/scala/io/youi/optimizer/HTMLOptimizer.scala
@@ -3,6 +3,7 @@ package io.youi.optimizer
 import java.io.{File, FileNotFoundException}
 import java.net.URL
+  
+import com.roundeights.hasher.Hasher
 import io.youi.stream._
 import org.powerscala.io._
@@ -19,10 +20,10 @@ object HTMLOptimizer {
    * Cleans up the HTML removing all JavaScript (inline and external) creating files for them.
    *
    * @param input the original HTML file to be optimized
+  
+    * @param baseDirectory the base directory to look up JavaScript paths from
    * @return Optimized
    */
-  
-  private def stage1(input: File): Optimized = {
-  
-    val directory = input.getParentFile
+  
+  private def stage1(input: File, baseDirectory: File): Optimized = {
    val stream = HTMLParser(input)
    var scripts = ListBuffer.empty[ScriptFile]
    val result = stream.stream(List(
@@ -52,7 +53,7 @@ object HTMLOptimizer {
              ScriptFile(file, map)
            } else {                                                                                  // Local script
              val map = if (minified) {
-  
-                val minifiedFile = new File(directory, s"$src.map")
+  
+                val minifiedFile = new File(baseDirectory, s"$src.map")
                if (minifiedFile.isFile) {
                  Some(minifiedFile)
                } else {
@@ -62,7 +63,7 @@ object HTMLOptimizer {
              } else {
                None
              }
-  
-              ScriptFile(new File(directory, src), map)
+  
+              ScriptFile(new File(baseDirectory, src), map)
            }
          }
          case None => {                // Inline script
@@ -137,20 +138,81 @@ object HTMLOptimizer {
    scribe.info(s"Saved(HTML: $htmlTrimmed, JavaScript: $jsTrimmed, Merged: ${optimized.scripts.size} JavaScript files)")
  }
-  
-  def optimize(directory: File, htmlPath: String, jsPath: String): Unit = {
-  
-    // TODO: Support incrementing js name if duplicates are found
-  
-    val html = new File(directory, htmlPath)
-  
-    val js = new File(directory, jsPath)
-  
-    val optimized = stage1(html)
-  
-    // TODO: detect if these scripts have been optimized already
-  
-    stage2(optimized)
-  
-    stage3(optimized, js)
-  
-    stage4(optimized.html, html, jsPath)
-  
-    stage5(optimized, html, js)
-  
-    // TODO: cache optimized for multiple runs
+  
+  private def nextJSFile(directory: File, jsPath: String, increment: Int = 0): File = {
+  
+    val name = jsPath.substring(0, jsPath.lastIndexOf('.'))
+  
+    val fileName = if (increment == 0) {
+  
+      s"$name.js"
+  
+    } else {
+  
+      s"$name$increment.js"
+  
+    }
+  
+    val file = new File(directory, fileName)
+  
+    if (!file.exists()) {
+  
+      file
+  
+    } else {
+  
+      nextJSFile(directory, jsPath, increment + 1)
+  
+    }
+  
+  }
+  
+
+  
+  private var cache = Map.empty[String, CachedOptimization]
+  
+
+  
+  def optimize(baseDirectory: File, directory: File, htmlPath: String, jsPath: String): Unit = synchronized {
+  
+    try {
+  
+      scribe.info(s"Optimizing $htmlPath...")
+  
+      val html = new File(directory, htmlPath)
+  
+      val optimized = stage1(html, baseDirectory)
+  
+      if (optimized.scripts.isEmpty) {
+  
+        scribe.info(s"No JavaScript in $htmlPath, skipping...")
+  
+      } else {
+  
+        val cached = cache.get(optimized.crc32) match {
+  
+          case Some(c) => {
+  
+            scribe.info(s"Using cached JavaScript file: ${c.js.getAbsolutePath}")
+  
+            c
+  
+          }
+  
+          case None => {
+  
+            val jsFile = nextJSFile(baseDirectory, jsPath)
+  
+            scribe.info(s"Creating new JavaScript file: ${jsFile.getAbsolutePath}")
+  
+            stage2(optimized)
+  
+            stage3(optimized, jsFile)
+  
+            CachedOptimization(optimized.scripts, jsFile, optimized.crc32)
+  
+          }
+  
+        }
+  
+        val js = cached.js
+  
+        val updatedJSPath = js.getCanonicalPath.substring(directory.getCanonicalPath.length + (if (jsPath.startsWith("/")) 0 else 1))
+  
+        stage4(optimized.html, html, updatedJSPath)
+  
+        stage5(optimized, html, js)
+  
+        cached.usedBy += htmlPath
+  
+        cache += optimized.crc32 -> cached
+  
+      }
+  
+    } catch {
+  
+      case t: Throwable => throw new RuntimeException(s"Error occurred while processing $htmlPath", t)
+  
+    }
  }
+  
+
+  
+  def outputStats(): Unit = {
+  
+    cache.values.foreach { cache =>
+  
+      scribe.info(s"${cache.js.getName} used by ${cache.usedBy.size} pages: ${cache.usedBy.mkString(", ")}")
+  
+    }
+  
+  }
+  
+
+  
+  final def optimizeDirectory(directory: File, jsPath: String, recursive: Boolean = true, baseDirectory: Option[File] = None): Unit = {
+  
+    directory.listFiles().foreach { file =>
+  
+      if (file.getName.toLowerCase.endsWith(".html")) {       // Found HTML file
+  
+        optimize(baseDirectory.getOrElse(directory), file.getParentFile, file.getName, jsPath)
+  
+      } else if (file.isDirectory && recursive) {             // Found directory
+  
+        optimizeDirectory(file, jsPath, recursive, baseDirectory.orElse(Some(directory)))
+  
+      }
+  
+    }
+  
+    scribe.info(s"Processing completed. Processed ${cache.values.flatMap(_.usedBy).size} and created ${cache.size} optimized JavaScript files:")
+  
+    outputStats()
+  
+  }
+  
+
+  
+  def dispose(): Unit = cache = Map.empty
+  
+}
+  
+
+  
+case class Optimized(html: File, scripts: List[ScriptFile], originalSize: Long) {
+  
+  lazy val crc32: String = scripts.map(sf => Hasher(sf.js).crc32.hex).mkString(",")
 }
-  
-case class Optimized(html: File, scripts: List[ScriptFile], originalSize: Long)
+  
+case class CachedOptimization(scripts: List[ScriptFile], js: File, crc32: String, var usedBy: Set[String] = Set.empty)
 case class ScriptFile(js: File, map: Option[File])
diff --git a/template/jvm/src/main/scala/io/youi/template/TemplateCompiler.scala b/template/jvm/src/main/scala/io/youi/template/TemplateCompiler.scala
@@ -2,6 +2,7 @@ package io.youi.template
 import java.io.File
+  
+import io.youi.optimizer.HTMLOptimizer
 import scribe.formatter.FormatterBuilder
 import scribe.{LogHandler, Logger}
 import io.youi.stream.{ByTag, Delta, HTMLParser}
@@ -16,7 +17,8 @@ class TemplateCompiler(val sourceDirectory: File,
                       val destinationDirectory: File,
                       val compressCSS: Boolean = false,
                       val removeDotHTML: Boolean = false,
-  
-                       val consoleCommands: Boolean = true) {
+  
+                       val consoleCommands: Boolean = true,
+  
+                       val optimize: Boolean = false) {
  private[template] var pages = Set.empty[String]
  private val server = new ServerTemplateApplication(this)
@@ -63,9 +65,6 @@ class TemplateCompiler(val sourceDirectory: File,
      deleteDestination()
    }
-  
-    // Generate pages
-  
-    compileAllPages()
-  
-
    // Copy assets
    copyAssets()
@@ -74,6 +73,9 @@ class TemplateCompiler(val sourceDirectory: File,
    // Compile SASS files
    compileAllSass()
+  
+
+  
+    // Generate pages
+  
+    compileAllPages()
  }
  private def processRecursively(directory: File)(handler: File => Unit): Unit = directory.listFiles.foreach { file =>
@@ -107,7 +109,10 @@ class TemplateCompiler(val sourceDirectory: File,
    destination.getParentFile.mkdirs()
    IO.stream(html, destination)
-  
-    // TODO: support JavaScript merging
+  
+    if (optimize) {
+  
+      HTMLOptimizer.optimize(destinationDirectory, destinationDirectory, fileName, "/js/optimized.js")
+  
+    }
+  
+
    // TODO: support CSS merging
    // TODO: support HTML minification
@@ -224,19 +229,22 @@ object LoggingProcessLogger extends ProcessLogger {
 }
 object TemplateCompiler {
-  
-  def main(args: Array[String]): Unit = if (args.length == 3) {
+  
+  def main(args: Array[String]): Unit = if (args.length == 4) {
    Logger.root.clearHandlers()
    Logger.root.addHandler(LogHandler(formatter = FormatterBuilder().date().string(" - ").message.newLine))
    val mode = args(0)
-  
-    val sourceDirectory = new File(args(1))
-  
-    val destinationDirectory = new File(args(2))
+  
+    val modification = args(1)
+  
+    val sourceDirectory = new File(args(2))
+  
+    val destinationDirectory = new File(args(3))
+  
+
+  
+    val optimize = modification == "optimize"
    assert(sourceDirectory.isDirectory, s"Source directory must be a directory (${sourceDirectory.getAbsolutePath})")
    assert(!destinationDirectory.isFile, s"Destination must be a directory, but found a file (${destinationDirectory.getAbsolutePath})")
    destinationDirectory.mkdirs()
-  
-    val compiler = new TemplateCompiler(sourceDirectory, destinationDirectory, removeDotHTML = true, consoleCommands = true)
+  
+    val compiler = new TemplateCompiler(sourceDirectory, destinationDirectory, removeDotHTML = true, consoleCommands = true, optimize = optimize)
    try {
      compiler.compileAll(deleteFirst = true)
      if (mode.equalsIgnoreCase("watch") || mode.equalsIgnoreCase("server")) {
@@ -263,11 +271,14 @@ object TemplateCompiler {
      }
    }
  } else {
-  
-    println("Usage: youi-template <mode> <source directory> <destination directory>")
+  
+    println("Usage: youi-template <mode> <modification> <source directory> <destination directory>")
    println("\t<mode> is one of the following options:")
    println("\t\tcompile - does a full compile and then exit")
    println("\t\twatch - does a full compile, then watches for changes and compiles on-demand")
    println("\t\tserver - does a full compile, then starts a server to serve the pages and will auto-reload the page on change")
+  
+    println("\t<modification> defines modifications that should take place against the compiled template and must be one of the following options:")
+  
+    println("\t\tnone - no optimizations or modifications are applied after compilation")
+  
+    println("\t\toptimize - merges all JavaScript into a single source file (including inline and remote scripts), optimizes it, minifies it, and obfuscates it while generating a js.map file for it")
    println("\t<source directory> is the location where the source files are stored. Supports the following sub-folders:")
    println("\t\tassets - files within this directory or copied as-is into the destination directory")
    println("\t\tless - looks for .less files to compile and put into the css directory of the destination")