Add Mustache templating as an optional step in the

Javascript compilation process. See the README for details.

Implementation notes:

  - Add some more tests.
  - Refactored code to move URL and file handling into ManifestObjects.
  - Removed complex download tasks and do the downloads whenever we compile.
  - Update buid.properties to the next snapshot version.

.ensime

@@ -0,0 +1,9 @@
+;; This config was generated using ensime-config-gen. Feel free to customize its contents manually.
+
+(
+
+:project-package "untyped"
+
+:use-sbt t
+
+)

README.markdown

@@ -20,7 +20,7 @@ content into it:
 
     class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
       val untypedRepo = "Untyped Repo" at "http://repo.untyped.com"
-      val closureCompiler = "untyped" % "sbt-closure" % "0.3-SNAPSHOT"
+      val closureCompiler = "untyped" % "sbt-closure" % "0.4"
     }
 
 This will give you the ability to use the plugin in your project file. For example:
@@ -66,6 +66,37 @@ method. See the source for details.
 Finally, you can execute the plugin's compilation step independently of
 `prepare-webapp` using `sbt compile-js`.
 
+Templating
+================
+
+It is sometime useful to template Javascript files. For example, you might want
+scripts to refer to localhost while developing and your live server when
+deployed. This plugin supports templating Javascript files using the [Mustache]
+format and [Lift style properties] (though the implementation has no dependency
+on Lift).
+
+In summary, properties are looked for in =src/main/resources/prop= (by default;
+see below for customization). They are in the standard Java format. If you
+aren't interested in changing your properties depending on your build
+configuration just place the properties in =default.props=. Otherwise property
+files should be named =modeName.props=, where modeName is the setting of the
+=run.mode= system property, which can take on values of =test=, =staging=,
+=production=, =pilot=, or =default=.. If =run.mode= is not set, =default= is
+assumed.
+
+Any Javascript file that contains =.template= will be passed through a Mustache
+template processor before being processed by the Google compiler.
+
+Parameters controlling templating are:
+
+   - =closureJsIsTemplated= is function that indicates if a given Javascript
+     file should be run through the template processor
+
+   - =closurePropertiesPath= determines where properties are found
+
+[Mustache]: http://mustache.github.com/
+[Lift style]: http://www.assembla.com/spaces/liftweb/wiki/Properties
+
 Acknowledgements
 ================
 

project/build.properties

@@ -3,6 +3,6 @@
 project.organization=untyped
 project.name=sbt-closure
 sbt.version=0.7.4
-project.version=0.4
+project.version=0.5-SNAPSHOT
 build.scala.versions=2.7.7
 project.initialize=false

src/main/scala/untyped/ClosureCompilerConfig.scala

@@ -0,0 +1,49 @@
+package untyped
+
+import com.google.javascript.jscomp._
+
+import sbt._
+
+trait ClosureCompilerConfig extends MavenStyleWebScalaPaths {
+
+  // Configuration ------------------------------
+
+  // This is a convenience method from Project.
+  def descendents(parent: PathFinder, include: FileFilter): PathFinder
+
+  /**
+   * Returns true if the path refers to a file that should be templated
+   *
+   * It is templated if the file name contains .template. E.g. foo.template.js
+   */
+  def closureJsIsTemplated(path: Path): Boolean = path.name.contains(".template")
+
+  /**
+   * Where we should look to find properties files that supply values we use when templating
+   */
+  def closurePropertiesPath: Path = mainResourcesPath
+
+  def closureSourcePath: Path = webappPath
+
+  def closureJsSourceFilter: NameFilter = GlobFilter("*.js")
+  def closureJsSources: PathFinder = descendents(closureSourcePath, closureJsSourceFilter)
+
+  def closureManifestSourceFilter: NameFilter = GlobFilter("*.jsm") | "*.jsmanifest"
+  def closureManifestSources: PathFinder = descendents(closureSourcePath, closureManifestSourceFilter)
+
+  def closureOutputPath: Path = (outputPath / "sbt-closure-temp") ##
+
+  var _closurePrettyPrint = false
+  def closurePrettyPrint = _closurePrettyPrint
+
+  var _closureVariableRenamingPolicy = VariableRenamingPolicy.LOCAL
+  def closureVariableRenamingPolicy = _closureVariableRenamingPolicy
+
+  def closureCompilerOptions = {
+    val options = new CompilerOptions
+    options.variableRenaming = closureVariableRenamingPolicy
+    options.prettyPrint = closurePrettyPrint
+    options
+  }  
+
+}

src/main/scala/untyped/ClosureCompilerPlugin.scala

@@ -13,49 +13,18 @@ import com.google.javascript.jscomp._
 import com.samskivert.mustache.{Mustache,Template}
 
 
-trait ClosureCompilerPlugin extends DefaultWebProject {
+trait ClosureCompilerPlugin extends DefaultWebProject with ClosureCompilerConfig {
 
   // Configuration ------------------------------
 
-  // Returns true if the path refers to a file that should be templated
-  // 
-  // It is templated if the file name contains .template. E.g. foo.template.js
-  def closureJsIsTemplated(path: Path): Boolean = path.name.contains(".template")
-
-  // Where we should look to find properties files that supply values we use when templating
-  def closurePropertiesPath: Path = mainResourcesPath
-
-  def closureSourcePath: Path = webappPath
-
-  def closureJsSourceFilter: NameFilter = filter("*.js")
-  def closureJsSources: PathFinder = descendents(closureSourcePath, closureJsSourceFilter)
-
-  def closureManifestSourceFilter: NameFilter = filter("*.jsm") | "*.jsmanifest"
-  def closureManifestSources: PathFinder = descendents(closureSourcePath, closureManifestSourceFilter)
-
-  def closureOutputPath: Path = (outputPath / "sbt-closure-temp") ##
-
-  var _closurePrettyPrint = false
-  def closurePrettyPrint = _closurePrettyPrint
-
-  var _closureVariableRenamingPolicy = VariableRenamingPolicy.LOCAL
-  def closureVariableRenamingPolicy = _closureVariableRenamingPolicy
-
-  def closureCompilerOptions = {
-    val options = new CompilerOptions
-    options.variableRenaming = closureVariableRenamingPolicy
-    options.prettyPrint = closurePrettyPrint
-    options
-  }
-
   log.debug("Closure compiler config:")
   log.debug("  - closureSourcePath           : " + closureSourcePath)
   log.debug("  - closureOutputPath           : " + closureOutputPath)
   log.debug("  - closureJsSourceFilter       : " + closureJsSourceFilter)
   log.debug("  - closureJsSources            : " + closureJsSources)
   log.debug("  - closureManifestSourceFilter : " + closureManifestSourceFilter)
   log.debug("  - closureManifestSources      : " + closureManifestSources)
-  
+
   // Top-level stuff ----------------------------
 
   lazy val compileJs = dynamic(compileJsAction) describedAs "Compiles Javascript manifest files"
@@ -98,87 +67,97 @@ trait ClosureCompilerPlugin extends DefaultWebProject {
     log.debug("JS manifest config:")
     log.debug("  - manifestPath : " + manifestPath)
     log.debug("  - outputPath   : " + outputPath)
-    log.debug("  - lines        : " + lines)
-    log.debug("  - urls         : " + urls)
-    log.debug("  - urlPaths     : " + urlPaths)
-    log.debug("  - sourcePaths  : " + sourcePaths)
+    log.debug("  - lines        : " + manifestObjects)
+    log.debug("  - urls         : " + urlObjects)
+//    log.debug("  - urlPaths     : " + urlPaths)
+//    log.debug("  - sourcePaths  : " + sourcePaths)
 
     // Reading the manifest ---------------------
+
+    import ManifestOps._
 
     // Before we can build a JS file, we have to read its manifest,
     // chop out comments, and skip blank lines:
 
-    def stripComments(line: String) = "#.*$".r.replaceAllIn(line, "").trim
-    def isSkippable(line: String): Boolean = stripComments(line) == ""
-    def isURL(line: String): Boolean = stripComments(line).matches("^https?:.*")
 
-    def lines: List[String] =
-      FileUtilities.readString(manifestPath.asFile, log).
-                    right.
-                    get.
-                    split("[\r\n]+").
-                    filter(item => !isSkippable(item)).
-                    toList
+    lazy val manifestObjects: List[ManifestObject] =
+      parse(FileUtilities.readString(manifestPath.asFile, log).
+                          right.
+                          get)
+
+    lazy val urlObjects: List[ManifestUrl] =
+      manifestObjects.foldRight(Nil: List[ManifestUrl]){(elt, lst) => 
+        elt match {
+          case e:ManifestUrl => e :: lst
+          case _ => lst
+        }}
+
 
     // URLs -------------------------------------
 
     // The first part of building a JS file is downloading and caching
     // any URLs specified in the manifest:
-
-    def urlToFilename(line: String): String = 
-      """[^A-Za-z0-9.]""".r.replaceAllIn(line, "_")
-
-    def urlContent(url: URL): String =
-      Source.fromInputStream(url.openStream).mkString
-
-    def linePath(line: String): Path = {
-      if(isURL(line)) {
-        toOutputPath(directoryPath) / urlToFilename(line)
-      } else {
-        Path.fromString(directoryPath, line)
-      }
-    }
+
+    // def urlLines: List[String] = lines.filter(isUrl _)
+    // def urls: List[URL] = urlLines.map(new URL(_))
+    // def urlPaths: List[Path] = urlLines.map(linePath _)
+
+    // Templating -------------------------------
 
-    def urlLines: List[String] = lines.filter(isURL _)
-    def urls: List[URL] = urlLines.map(new URL(_))
-    def urlPaths: List[Path] = urlLines.map(linePath _)
-
-    def download(url: URL, path: Path): Option[String] = {
-      FileUtilities.createDirectory(Path.fromFile(path.asFile.getParent), log).
-        orElse(FileUtilities.write(path.asFile, urlContent(url), log))
-    }
-
-    def downloadTasks: List[Task] = {
-      for((url, path) <- urls.zip(urlPaths)) yield {
-        val label = "closure-download " + url.toString
-        val product = List(path) from List(manifestPath)
-
-        fileTask(label, product){
-          log.debug("to " + path.toString)
-          download(url, path)
-        }.named(label)
+    /** Instantiate the properties used for Mustache templating */
+    val attributes: Properties = {
+      val props = new Props(closurePropertiesPath.asFile)
+      props.properties.getOrElse {
+        throw new RuntimeException("Closure Compiler Plugin: No properties found for processing Mustache templates. Looked in " + props.searchPaths)
       }
     }
-
-    // Templating -------------------------------
 
-    val attributes: Properties = new Props(closurePropertiesPath.asFile).properties.get
+    /**
+     * By default the JMustache implementation will treat
+     * variables named like.this as a two part name and look
+     * for a variable called this within one called like
+     * (called compound variables in the docs). This breaks
+     * things with the default naming conventions for
+     * Java/Lift properties so we turn it off.
+     */
+    lazy val compiler = Mustache.compiler().standardsMode(true)
 
     def renderTemplate(path: Path): String = {
-      val tmpl = 
-        Mustache.compiler().compile(new BufferedReader(new FileReader(path.asFile)))
+      val tmpl = compiler.compile(new BufferedReader(new FileReader(path.asFile)))
       tmpl.execute(attributes)
-
     }
 
+    def download(url: ManifestUrl, path: Path): Unit =
+      FileUtilities.createDirectory(Path.fromFile(path.asFile.getParent), log) match {
+        case Some(errorMsg) =>
+          throw new Exception("Failed to download " + url.url + ": " + errorMsg)
+
+        case None =>
+          FileUtilities.write(path.asFile, url.content, log)
+      }
+
     // Compilation ------------------------------
 
-    // Once URLs have been downloaded and cached, we can run the whole file
-    // through the Closure compiler:
+    // Once URLs have been downloaded and cached, we
+    // concatenate everything into one big file and run it
+    // through the Closure compiler
+
+    def objectPath(obj: ManifestObject): Path =
+      obj match {
+        case file: ManifestFile =>
+          file.path(directoryPath)
+
+        // We download and cache ManifestUrls lines in a staging directory
+        case url: ManifestUrl =>
+          val outputDir = toOutputPath(directoryPath)
+          val outputPath = url.path(outputDir)
+          download(url, outputPath)
+          outputPath
+      }
 
     def externPaths: List[Path] = Nil
 
-    def sourcePaths: List[Path] = lines.map(linePath _)
+    def sourcePaths: List[Path] = manifestObjects.map(objectPath _)
 
     def pathToJSSourceFile(path: Path): JSSourceFile =
       if(closureJsIsTemplated(path)) 
@@ -221,7 +200,7 @@ trait ClosureCompilerPlugin extends DefaultWebProject {
       fileTask(label, product){
         log.debug("to " + outputPath.toString)
         compile
-      }.named(label).dependsOn(downloadTasks : _*)
+      }.named(label)
     }
 
   }

src/main/scala/untyped/ManifestObject.scala

@@ -0,0 +1,21 @@
+package untyped
+
+import java.net.URL
+import scala.io.Source
+import sbt._
+
+sealed abstract class ManifestObject {
+  def filename: String
+  def path(parent: Path): Path = 
+    filename.split("""[/\\]""").foldLeft(parent)(_ / _)
+}
+
+case class ManifestFile(val filename: String) extends ManifestObject 
+
+case class ManifestUrl(val url: String) extends ManifestObject {
+  lazy val filename: String = """[^A-Za-z0-9.]""".r.replaceAllIn(url, "_")
+
+  def content: String = Source.fromInputStream(new URL(url).openStream).mkString
+}
+
+

src/main/scala/untyped/ManifestOps.scala

@@ -0,0 +1,17 @@
+package untyped
+
+import java.io.File
+
+object ManifestOps {
+  def stripComments(line: String) = "#.*$".r.replaceAllIn(line, "").trim
+  def isSkippable(line: String): Boolean = stripComments(line) == ""
+  def isUrl(line: String): Boolean = stripComments(line).matches("^https?:.*")
+
+  def parse(manifest: String): List[ManifestObject] = 
+    manifest.split("[\r\n]+").
+             map(stripComments _).
+             filter(item => !isSkippable(item)).
+             map(line => if(isUrl(line)) ManifestUrl(line) else ManifestFile(line)).
+             toList
+
+}

src/main/scala/untyped/Props.scala

@@ -57,7 +57,7 @@ class Props(val basePath: File) {
   }
 
   lazy val properties: Option[Properties] = {
-    searchPaths.find(p => {println(p); new File(p).exists()}).map{ propFile =>
+    searchPaths.find(p => new File(p).exists()).map{ propFile =>
       val props = new Properties()
       props.load(new FileInputStream(new File(propFile)))
       props