Permalink
Browse files

[split] Squashed commit of the following:

commit faf3d257e7dc95d1492b7c8c6faf9da3c53adac6
Merge: d2ecdbd 53fdb61
Author: mmcbride <mccv@twitter.com>
Date:   Mon Aug 8 22:05:09 2011 -0700

    Merge branch 'master' into persistent_eval

commit d2ecdbddd288607cd1fb597ba04685628ad8b2f0
Author: mmcbride <mccv@twitter.com>
Date:   Mon Aug 8 21:40:32 2011 -0700

    two minor things from jeremy cloud

commit a17260bd1cd90bab0ac635e0257a9bea8e9da62f
Merge: 5f26ee0 e8d277e
Author: mmcbride <mccv@twitter.com>
Date:   Wed Aug 3 20:16:06 2011 -0700

    Merge branch 'master' into persistent_eval

commit 5f26ee0544ad39439c509e8b87e60c4cff0f07b9
Author: mmcbride <mccv@twitter.com>
Date:   Wed Aug 3 20:13:33 2011 -0700

    feedback from RB

commit 775c8661f485724af3be51be720b821b83b044ec
Merge: 350c313 f7d1db2
Author: mmcbride <mccv@twitter.com>
Date:   Tue Jul 26 12:41:00 2011 -0700

    Merge branch 'master' into persistent_eval

commit 350c313991b31f9e332753a62ff5b92c61ce7cae
Author: mmcbride <mccv@twitter.com>
Date:   Tue Jul 26 10:58:25 2011 -0700

    persistent compilation targets for eval
  • Loading branch information...
1 parent 0b6b18d commit 6888fb57bd6b5c547ccee0e9dd5d0ece2bf7eae6 mmcbride committed Aug 9, 2011
@@ -27,10 +27,12 @@ import scala.collection.mutable
import scala.io.Source
import scala.tools.nsc.{Global, Settings}
import scala.tools.nsc.interpreter.AbstractFileClassLoader
-import scala.tools.nsc.io.VirtualDirectory
+import scala.tools.nsc.io.{AbstractFile, VirtualDirectory}
import scala.tools.nsc.reporters.AbstractReporter
import scala.tools.nsc.util.{BatchSourceFile, Position}
+case class LastMod(timestamp: Long, code: String)
+
/**
* Evaluate a file or string and return the result.
*/
@@ -39,7 +41,35 @@ object Eval extends Eval {
private val jvmId = java.lang.Math.abs(new Random().nextInt())
}
-class Eval {
+/**
+ * evaluates files, strings or input streams, and returns the result.
+ * In all cases, code to be evaled is wrapped in an apply method in a
+ * generated class. An instance of the class is instantiated, and the
+ * result of apply is returned.
+ *
+ * If target is None, the results are compiled to memory (and are therefore
+ * ephemeral). If target is Some(path), path must point to a directory, and
+ * eval emits class files to that directory.
+ *
+ * eval also supports a limited set of preprocessors. Limited means
+ * exactly one, that supports directives of the form #include <file>.
+ *
+ * The general flow of evaluation is
+ * # convert arguments to a string
+ * # run preprocessors on that string
+ * # wrap processed code in a class
+ * # compile the class
+ * # create an instance of that class
+ * # return the results of apply()
+ */
+class Eval(target: Option[File]) {
+ /**
+ * empty constructor for backwards compatibility
+ */
+ def this() {
+ this(None)
+ }
+
import Eval.jvmId
private lazy val compilerPath = try {
@@ -57,9 +87,13 @@ class Eval {
}
/**
- * Preprocessors to run the code through before it is passed to the Scala compiler
+ * Preprocessors to run the code through before it is passed to the Scala compiler.
+ * if you want to add new resolvers, you can do so with
+ * new Eval(...) {
+ * lazy val preprocessors = {...}
+ * }
*/
- private lazy val preprocessors: Seq[Preprocessor] =
+ protected lazy val preprocessors: Seq[Preprocessor] =
Seq(
new IncludePreprocessor(
Seq(
@@ -70,45 +104,95 @@ class Eval {
)
)
- private lazy val compiler = new StringCompiler(2, preprocessors)
+ private lazy val compiler = new StringCompiler(2, target)
+
+ /**
+ * run preprocessors on our string, returning a LastMod
+ * where timestamp is the last modified time of any file in that contributed
+ * to the text.
+ * Last modified is computed here because we support includes
+ */
+ def sourceForString(code: String) = {
+ preprocessors.foldLeft(LastMod(0L, code)) { (acc, p) =>
+ val processed = p(acc.code)
+ LastMod(acc.timestamp max processed.timestamp, processed.code)
+ }
+ }
/**
* Eval[Int]("1 + 1") // => 2
*/
def apply[T](code: String, resetState: Boolean = true): T = {
- val id = uniqueId(code)
- val className = "Evaluator__" + id
- val cls = compiler(wrapCodeInClass(className, code), className, id, resetState)
- cls.getConstructor().newInstance().asInstanceOf[() => Any].apply().asInstanceOf[T]
+ val processed = sourceForString(code)
+ applyProcessed(processed.code, resetState)
}
/**
* Eval[Int](new File("..."))
*/
def apply[T](files: File*): T = {
- apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n"))
+ if (target.isDefined) {
+ val targetDir = target.get
+ val unprocessedSource = files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n")
+ val processed = sourceForString(unprocessedSource)
+ val oldestTarget = targetDir.listFiles.foldLeft(Long.MaxValue)((oldest, f) => f.lastModified min oldest)
+ if (processed.timestamp > oldestTarget) {
+ compiler.reset()
+ }
+ val className = "Evaluator__" + files(0).getName.split("\\.")(0)
+ applyProcessed(className, processed.code, false)
+ } else {
+ apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n"), true)
+ }
}
/**
* Eval[Int](getClass.getResourceAsStream("..."))
*/
def apply[T](stream: InputStream): T = {
- apply(Source.fromInputStream(stream).mkString)
+ apply(sourceForString(Source.fromInputStream(stream).mkString).code)
+ }
+
+ /**
+ * same as apply[T], but does not run preprocessors.
+ * Will generate a classname of the form Evaluater__<unique>,
+ * where unique is computed from the jvmID (a random number)
+ * and a digest of code
+ */
+ def applyProcessed[T](code: String, resetState: Boolean): T = {
+ val id = uniqueId(code)
+ val className = "Evaluator__" + id
+ applyProcessed(className, code, resetState)
}
+ /**
+ * same as apply[T], but does not run preprocessors.
+ */
+ def applyProcessed[T](className: String, code: String, resetState: Boolean): T = {
+ val cls = compiler(wrapCodeInClass(className, code), className, resetState)
+ cls.getConstructor().newInstance().asInstanceOf[() => Any].apply().asInstanceOf[T]
+ }
+
+ /**
+ * converts the given file to evaluable source.
+ * delegates to toSource(code: String)
+ */
def toSource(file: File): String = {
toSource(scala.io.Source.fromFile(file).mkString)
}
+ /**
+ * converts the given file to evaluable source.
+ */
def toSource(code: String): String = {
- compiler.sourceForString(code)
+ sourceForString(code).code
}
/**
* Compile an entire source file into the virtual classloader.
*/
def compile(code: String) {
- compiler(code)
+ compiler(sourceForString(code).code)
}
/**
@@ -124,7 +208,7 @@ class Eval {
* @throw CompilerException if not Eval-able.
*/
def check(code: String) {
- val id = uniqueId(code)
+ val id = uniqueId(sourceForString(code).code)
val className = "Evaluator__" + id
val wrappedCode = wrapCodeInClass(className, code)
compile(wrappedCode) // may throw CompilerException
@@ -203,12 +287,13 @@ class Eval {
}
trait Preprocessor {
- def apply(code: String): String
+ def apply(code: String): LastMod
}
trait Resolver {
def resolvable(path: String): Boolean
def get(path: String): InputStream
+ def lastModified(path: String): Long
}
class FilesystemResolver(root: File) extends Resolver {
@@ -219,6 +304,14 @@ class Eval {
def resolvable(path: String): Boolean =
file(path).exists
+ def lastModified(path: String): Long = {
+ if (resolvable(path)) {
+ file(path).lastModified
+ } else {
+ 0
+ }
+ }
+
def get(path: String): InputStream =
new FileInputStream(file(path))
}
@@ -230,6 +323,8 @@ class Eval {
def resolvable(path: String): Boolean =
clazz.getResourceAsStream(quotePath(path)) != null
+ def lastModified(path: String): Long = 0
+
def get(path: String): InputStream =
clazz.getResourceAsStream(quotePath(path))
}
@@ -245,40 +340,51 @@ class Eval {
* @example #include file-name.scala
*
* This is the only directive supported by this preprocessor.
+ *
+ * Note that it is *not* recursive. Included files cannot have includes
*/
- private class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor {
- def apply(code: String): String =
- code.lines map { line: String =>
+ class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor {
+ def apply(code: String): LastMod = {
+ var lastMod = 0L
+ val lines = code.lines map { line: String =>
val tokens = line.trim.split(' ')
if (tokens.length == 2 && tokens(0).equals("#include")) {
val path = tokens(1)
resolvers find { resolver: Resolver =>
resolver.resolvable(path)
} match {
- case Some(r: Resolver) =>
+ case Some(r: Resolver) => {
+ lastMod = lastMod max r.lastModified(path)
StreamIO.buffer(r.get(path)).toString
+ }
case _ =>
throw new IllegalStateException("No resolver could find '%s'".format(path))
}
} else {
line
}
- } mkString("\n")
+ }
+ val processed = lines.mkString("\n")
+ LastMod(lastMod, processed)
+ }
}
/**
* Dynamic scala compiler. Lots of (slow) state is created, so it may be advantageous to keep
* around one of these and reuse it.
*/
- private class StringCompiler(lineOffset: Int, preprocessors: Seq[Preprocessor]) {
- val virtualDirectory = new VirtualDirectory("(memory)", None)
+ private class StringCompiler(lineOffset: Int, targetDir: Option[File]) {
+ val target = targetDir match {
+ case Some(dir) => AbstractFile.getDirectory(dir)
+ case None => new VirtualDirectory("(memory)", None)
+ }
val cache = new mutable.HashMap[String, Class[_]]()
val settings = new Settings
settings.deprecation.value = true // enable detailed deprecation warnings
settings.unchecked.value = true // enable detailed unchecked warnings
- settings.outputDirs.setSingleOutput(virtualDirectory)
+ settings.outputDirs.setSingleOutput(target)
val pathList = compilerPath ::: libPath
settings.bootclasspath.value = pathList.mkString(File.pathSeparator)
@@ -321,12 +427,24 @@ class Eval {
* Class loader for finding classes compiled by this StringCompiler.
* After each reset, this class loader will not be able to find old compiled classes.
*/
- var classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader)
+ var classLoader = new AbstractFileClassLoader(target, this.getClass.getClassLoader)
def reset() {
- virtualDirectory.clear
+ targetDir match {
+ case None => {
+ target.asInstanceOf[VirtualDirectory].clear
+ }
+ case Some(t) => {
+ target.foreach { abstractFile =>
+ if (abstractFile.file == null || abstractFile.file.getName.endsWith(".class")) {
+ abstractFile.delete
+ }
+ }
+ }
+ }
+ cache.clear()
reporter.reset
- classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader)
+ classLoader = new AbstractFileClassLoader(target, this.getClass.getClassLoader)
}
object Debug {
@@ -362,14 +480,12 @@ class Eval {
* Compile scala code. It can be found using the above class loader.
*/
def apply(code: String) {
- val processedCode = sourceForString(code)
-
if (Debug.enabled)
- Debug.printWithLineNumbers(processedCode)
+ Debug.printWithLineNumbers(code)
// if you're looking for the performance hit, it's 1/2 this line...
val compiler = new global.Run
- val sourceFiles = List(new BatchSourceFile("(inline)", processedCode))
+ val sourceFiles = List(new BatchSourceFile("(inline)", code))
// ...and 1/2 this line:
compiler.compileSources(sourceFiles)
@@ -378,14 +494,10 @@ class Eval {
}
}
- def sourceForString(code: String) = {
- preprocessors.foldLeft(code) { case (c: String, p: Preprocessor) => p(c) }
- }
-
/**
* Compile a new class, load it, and return it. Thread-safe.
*/
- def apply(code: String, className: String, id: String, resetState: Boolean = true): Class[_] = {
+ def apply(code: String, className: String, resetState: Boolean = true): Class[_] = {
synchronized {
if (resetState) reset()
findClass(className).getOrElse {
@@ -22,6 +22,40 @@ object EvalSpec extends Specification {
derived() mustEqual "hello"
}
+ "apply(new File(...) with target" in {
+ val f = File.createTempFile("eval", "target")
+ f.delete()
+ f.mkdir()
+ val e = new Eval(Some(f))
+ val sourceFile = TempFile.fromResourcePath("/OnePlusOne.scala")
+ val res: Int = e(sourceFile)
+ res mustEqual 2
+
+ // make sure it created a class file with the expected name
+ val targetFileName = f.getAbsolutePath() + File.separator + "Evaluator__" + sourceFile.getName + ".class"
+ val targetFile = new File(targetFileName)
+ targetFile.exists must be_==(true)
+ val targetMod = targetFile.lastModified
+
+ // eval again, make sure it works
+ val res2: Int = e(sourceFile)
+ // and make sure it didn't create a new file
+ f.listFiles.length mustEqual 1
+ // and make sure it didn't update the file
+ val targetFile2 = new File(targetFileName)
+ targetFile2.lastModified mustEqual targetMod
+
+ // touch source, ensure recompile
+ sourceFile.setLastModified(System.currentTimeMillis())
+ val res3: Int = e(sourceFile)
+ res3 mustEqual 2
+ // and make sure it didn't create a different file
+ f.listFiles.length mustEqual 1
+ // and make sure it updated the file
+ val targetFile3 = new File(targetFileName)
+ targetFile3.lastModified must be_>=(targetMod)
+ }
+
"apply(InputStream)" in {
(new Eval).apply[Int](getClass.getResourceAsStream("/OnePlusOne.scala")) mustEqual 2
}

0 comments on commit 6888fb5

Please sign in to comment.