Browse files

[project @ robey@lag.net-20080128063218-3m59nfz5h3xn9y1q]

first checkpoint: can lex the basic tokens, and proves so with some unit
tests.
  • Loading branch information...
0 parents commit b29e1be97c8c928c9aaf8fbc7cceb299578ffbd8 Robey Pointer committed Jan 27, 2008
8 .classpath
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="src" output="build-test" path="test"/>
+ <classpathentry kind="con" path="ch.epfl.lamp.sdt.launching.SCALA_CONTAINER"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry kind="output" path="build"/>
+</classpath>
18 .project
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>configgy</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>ch.epfl.lamp.sdt.core.scalabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>ch.epfl.lamp.sdt.core.scalanature</nature>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
45 build.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project name="configgy" default="default" basedir=".">
+ <description>Builds, tests, and runs the project configgy.</description>
+ <import file="nbproject/build-impl.xml"/>
+
+ <property name="test.dir" value="build-test" />
+ <property name="docs.dir" value="docs" />
+
+ <!-- why doesn't scala.home get imported from build-impl? -->
+ <property environment="env" />
+ <property name="scala.home" value="${env.SCALA_HOME}" />
+ <property name="scala-compiler.jar" value="${scala.home}/lib/scala-compiler.jar" />
+ <property name="scala-library.jar" value="${scala.home}/lib/scala-library.jar" />
+ <path id="scala.classpath">
+ <pathelement location="${scala-compiler.jar}" />
+ <pathelement location="${scala-library.jar}" />
+ </path>
+ <taskdef resource="scala/tools/ant/antlib.xml">
+ <classpath refid="scala.classpath" />
+ </taskdef>
+
+ <!-- scalatests doesn't work with scala 2.6.2 yet, so gotta wait.. -->
+ <target name="compile-tests" depends="compile">
+ <scalac srcdir="test" destdir="${test.dir}" force="changed">
+ <classpath>
+ <path refid="scala.classpath" />
+ <pathelement location="${build.dir}" />
+ </classpath>
+ <include name="**/*.scala" />
+ </scalac>
+ </target>
+
+ <target name="test" depends="compile-tests">
+ <java classname="scala.tools.nsc.MainGenericRunner" fork="true">
+ <classpath>
+ <path refid="scala.classpath" />
+ <pathelement location="${build.dir}" />
+ <pathelement location="${test.dir}" />
+ </classpath>
+ <arg line="net.lag.configgy.TestRunner" />
+ </java>
+ </target>
+
+
+</project>
89 nbproject/build-impl.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+*** GENERATED FROM project.xml - DO NOT EDIT ***
+*** EDIT ../build.xml INSTEAD ***
+-->
+<project name="configgy-impl" default="default" basedir=".." xmlns:scalaProject="http://www.netbeans.org/ns/scala-project/1">
+ <target name="default" depends="compile"/>
+ <!--
+ ======================
+ INITIALIZATION SECTION
+ ======================
+ -->
+ <target name="-pre-init">
+ <!-- Empty placeholder for easier customization. -->
+ <!-- You can override this target in the ../build.xml file. -->
+ </target>
+ <target name="-init-properties" depends="-pre-init">
+ <property file="nbproject/private/private.properties"/>
+ <property file="nbproject/project.properties"/>
+ <property environment="env"/>
+ <condition property="scala.home" value="${env.SCALA_HOME}">
+ <isset property="env.SCALA_HOME"/>
+ </condition>
+ <fail unless="scala.home">
+ You must set SCALA_HOME or environment property or scala.home
+ property in nbproject/private/private.properties to point to
+ Scala installation directory.
+ </fail>
+ <property name="build.dir" value="build"/>
+ </target>
+ <target name="init" depends="-pre-init,-init-properties"/>
+ <!--
+ ===================
+ COMPILATION SECTION
+ ===================
+ -->
+ <target name="-pre-pre-compile" depends="init">
+ <mkdir dir="${build.dir}"/>
+ </target>
+ <target name="compile" depends="init,-pre-pre-compile">
+ <taskdef resource="scala/tools/ant/antlib.xml">
+ <classpath>
+ <pathelement location="${scala.home}/lib/scala-compiler.jar"/>
+ <pathelement location="${scala.home}/lib/scala-library.jar"/>
+ </classpath>
+ </taskdef>
+ <scalac srcdir="src" destdir="${build.dir}">
+ <classpath>
+ <pathelement location="${scala.home}/lib/scala-library.jar"/>
+ </classpath>
+ </scalac>
+ </target>
+ <target name="compile-single" depends="init,-pre-pre-compile">
+ <taskdef resource="scala/tools/ant/antlib.xml">
+ <classpath>
+ <pathelement location="${scala.home}/lib/scala-compiler.jar"/>
+ <pathelement location="${scala.home}/lib/scala-library.jar"/>
+ </classpath>
+ </taskdef>
+ <scalac srcdir="src" destdir="${build.dir}">
+ <classpath>
+ <pathelement location="${scala.home}/lib/scala-library.jar"/>
+ </classpath>
+ <patternset includes="${javac.includes}"/>
+ </scalac>
+ </target>
+ <!--
+ ===============
+ EXECUTION SECTION
+ ===============
+ -->
+ <target name="run-single" depends="init,compile-single">
+ <fail unless="run.class">Must select one file in the IDE or set run.class</fail>
+ <java fork="true" classname="${run.class}">
+ <classpath>
+ <path path="${build.dir}"/>
+ <pathelement location="${scala.home}/lib/scala-library.jar"/>
+ </classpath>
+ </java>
+ </target>
+ <!--
+ ===============
+ CLEANUP SECTION
+ ===============
+ -->
+ <target name="clean" depends="init">
+ <delete dir="${build.dir}"/>
+ </target>
+</project>
8 nbproject/genfiles.properties
@@ -0,0 +1,8 @@
+build.xml.data.CRC32=df4d828d
+build.xml.script.CRC32=237f52c1
+build.xml.stylesheet.CRC32=60f9ee0b
+# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
+# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
+nbproject/build-impl.xml.data.CRC32=df4d828d
+nbproject/build-impl.xml.script.CRC32=8aa23312
+nbproject/build-impl.xml.stylesheet.CRC32=a9c71f9f
2 nbproject/private/private.properties
@@ -0,0 +1,2 @@
+file.reference.configgy-src=/Users/robey/code/scala/configgy/src
+user.properties.file=/Users/robey/.netbeans/6.0/build.properties
58 nbproject/project.properties
@@ -0,0 +1,58 @@
+build.classes.dir=${build.dir}/classes
+build.classes.excludes=**/*.java,**/*.form
+# This directory is removed when the project is cleaned:
+build.dir=build
+build.generated.dir=${build.dir}/generated
+# Only compile against the classpath explicitly listed here:
+build.sysclasspath=ignore
+build.test.classes.dir=${build.dir}/test/classes
+build.test.results.dir=${build.dir}/test/results
+debug.classpath=\
+ ${run.classpath}
+debug.test.classpath=\
+ ${run.test.classpath}
+# This directory is removed when the project is cleaned:
+dist.dir=dist
+dist.jar=${dist.dir}/configgy.jar
+dist.javadoc.dir=${dist.dir}/javadoc
+excludes=
+file.reference.configgy-src=../src
+includes=**
+jar.compress=false
+javac.classpath=
+# Space-separated list of extra javac options
+javac.compilerargs=
+javac.deprecation=false
+javac.source=1.5
+javac.target=1.5
+javac.test.classpath=\
+ ${javac.classpath}:\
+ ${build.classes.dir}:\
+ ${libs.junit.classpath}:\
+ ${libs.junit_4.classpath}
+javadoc.additionalparam=
+javadoc.author=false
+javadoc.encoding=${source.encoding}
+javadoc.noindex=false
+javadoc.nonavbar=false
+javadoc.notree=false
+javadoc.private=false
+javadoc.splitindex=true
+javadoc.use=true
+javadoc.version=false
+javadoc.windowtitle=
+main.class=
+meta.inf.dir=${src.dir}/META-INF
+platform.active=default_platform
+run.classpath=\
+ ${javac.classpath}:\
+ ${build.classes.dir}
+# Space-separated list of JVM arguments used when running the project
+# (you may also define separate properties like run-sys-prop.name=value instead of -Dname=value
+# or test-sys-prop.name=value to set system properties for unit tests):
+run.jvmargs=
+run.test.classpath=\
+ ${javac.test.classpath}:\
+ ${build.test.classes.dir}
+source.encoding=UTF-8
+src.dir=${file.reference.configgy-src}
9 nbproject/project.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://www.netbeans.org/ns/project/1">
+ <type>org.netbeans.modules.scala.project</type>
+ <configuration>
+ <data xmlns="http://www.netbeans.org/ns/scala-project/1">
+ <name>configgy</name>
+ </data>
+ </configuration>
+</project>
105 src/net/lag/configgy/ConfigLexer.scala
@@ -0,0 +1,105 @@
+package net.lag.configgy
+
+import java.util.regex.Pattern
+
+import scala.util.parsing.combinator._
+import scala.util.parsing.combinator.syntactical.TokenParsers
+import scala.util.parsing.combinator.lexical.Lexical
+import scala.util.parsing.input.{CharArrayReader, Reader}
+import scala.util.parsing.input.CharArrayReader.EofCh
+import scala.util.parsing.syntax.Tokens
+
+
+class ConfigLexer extends Lexical with Tokens {
+
+ case class Number(value: String) extends Token {
+ def chars = value
+ override def toString = "Number(" + value + ")"
+ }
+
+ case class Ident(name: String) extends Token {
+ def chars = name
+ override def toString = "Ident(" + name + ")"
+ }
+
+ case class OpenTag(name: String) extends Token {
+ def chars = "<" + name + ">"
+ override def toString = "OpenTag(" + name + ")"
+ }
+
+ case class CloseTag(name: String) extends Token {
+ def chars = "</" + name + ">"
+ override def toString = "CloseTag(" + name + ")"
+ }
+
+ case class Assign extends Token {
+ def chars = "="
+ override def toString = "Assign"
+ }
+
+ case class CondAssign extends Token {
+ def chars = "?="
+ override def toString = "CondAssign"
+ }
+
+ case class QuotedString(value: String) extends Token {
+ def chars = value
+ override def toString = "QuotedString(\"" + StringUtils.quoteC(value) + "\")"
+ }
+
+
+ // helper function for turning match combos into strings
+ def pack(x: Any): String = x match {
+ case s: String => s
+ case c: Char => c.toString
+ case l: List[_] => (for (item <- l) yield pack(item)).mkString("")
+ case a ~ b => pack(a) + pack(b)
+ case Some(value) => pack(value)
+ case None => ""
+ }
+
+
+ val empties = Set[Char]() ++ " \t\n\r".toArray
+ override def whitespace = rep((elem("whitespace", empties.contains(_)) +) |
+ ('#' ~ rep(chrExcept('\n')) ~ '\n'))
+
+ override def token: Parser[Token] = number | ident | openTag | closeTag | assign | condAssign | quotedString | fini
+
+ def any = elem("any", ch => true)
+ val hexDigits = Set[Char]() ++ "0123456789abcdefABCDEF".toArray
+ def hexDigit = elem("hex digit", hexDigits.contains(_))
+
+ def number = opt(elem('-')) ~ rep1(digit) ~ opt(elem('.') ~ rep(digit)) ^^ { x: Any => new Number(pack(x)) }
+ def segment = (letter | elem('_')) ~ rep(letter | digit | elem('-') | elem('_')) ^^ { pack(_) }
+ def ident = segment ~ rep(elem('.') ~ segment) ^^ { x: Any => new Ident(pack(x)) }
+
+ def tagName = letter ~ rep(letter | digit | elem('-') | elem('.') | elem('_')) ^^ { pack(_) }
+ def openTag = '<' ~ tagName ~ '>' ^^ { new OpenTag(_) }
+ def closeTag = '<' ~ '/' ~ tagName ~ '>' ^^ { new CloseTag(_) }
+
+ def assign = elem('=') ^^ { case _ => new Assign }
+ def condAssign = elem('?') ~ elem('=') ^^ { case a ~ b => new CondAssign }
+
+ def quotedString = '"' ~ rep(chrExcept('\\', '"') | (elem('\\') ~ chrExcept('u', 'x')) |
+ (elem('\\') ~ elem('\n')) | (elem('\\') ~ elem('u') ~ repN(4, hexDigit)) |
+ (elem('\\') ~ elem('x') ~ repN(2, hexDigit))) ~ '"' ^^ { x: Any => new QuotedString(StringUtils.unquoteC(pack(x))) }
+
+ def fini = EofCh ^^ EOF
+
+ // for unit tests: scan a string and return a list of Tokens
+ def scan(s: String): List[Token] = {
+ scan(new Scanner(s))
+ }
+ def scan(scanner: Scanner): List[Token] = {
+ if (scanner.atEnd) {
+ List[Token](scanner.first)
+ } else {
+ scanner.first :: scan(scanner.rest)
+ }
+ }
+}
+
+class ConfigParser extends TokenParsers {
+ type Tokens = ConfigLexer
+ val lexical = new Tokens
+}
76 src/net/lag/configgy/StringUtils.scala
@@ -0,0 +1,76 @@
+package net.lag.configgy
+
+import java.util.regex._
+
+
+object StringUtils {
+ /*
+ * Scala does not yet (Dec 2007) support java's String.format natively.
+ * Fake it by building the Object[] manually for a handful of params.
+ */
+ def format(s: String, items: Any*) = String.format(s, (for (i <- items) yield i.asInstanceOf[Object]).toArray)
+
+
+ // emulate python re.sub
+ def patternSub(p: Pattern, s: String, replace: (Matcher => String)) = {
+ var offset = 0
+ var out = new StringBuilder
+ val m = p.matcher(s)
+
+ while (m.find()) {
+ if (m.start > offset) {
+ out.append(s.substring(offset, m.start))
+ }
+
+ out.append(replace(m))
+ offset = m.end
+ }
+
+ if (offset < s.length) {
+ out.append(s.substring(offset))
+ }
+ out.toString
+ }
+
+
+ private val QUOTE_RE = Pattern.compile("[\u0000-\u001f\u007f-\uffff\\\\\"]")
+
+ def quoteC(s: String) = {
+ patternSub(QUOTE_RE, s, (m: Matcher) => {
+ m.group.charAt(0) match {
+ case '\r' => "\\r"
+ case '\n' => "\\n"
+ case '\t' => "\\t"
+ case '"' => "\\\""
+ case '\\' => "\\\\"
+ case c => {
+ if (c <= 255) {
+ format("\\x%02x", c.asInstanceOf[Int])
+ } else {
+ format("\\u%04x", c.asInstanceOf[Int])
+ }
+ }
+ }
+ })
+ }
+
+
+ private val UNQUOTE_RE = Pattern.compile("\\\\(u[\\dA-Fa-f]{4}|x[\\dA-Fa-f]{2}|[^ux])")
+
+ def unquoteC(s: String) = {
+ patternSub(UNQUOTE_RE, s, (m: Matcher) => {
+ val ch = m.group(1).charAt(0) match {
+ // holy crap! this is terrible:
+ case 'u' => Character.valueOf(Integer.valueOf(m.group(1).substring(1), 16).asInstanceOf[Int].toChar)
+ case 'x' => Character.valueOf(Integer.valueOf(m.group(1).substring(1), 16).asInstanceOf[Int].toChar)
+ case 'r' => '\r'
+ case 'n' => '\n'
+ case 't' => '\t'
+ case x => x
+ }
+ ch.toString
+ })
+ }
+
+ // FIXME: what?
+}
31 test/net/lag/configgy/ConfigLexerTests.scala
@@ -0,0 +1,31 @@
+package net.lag.configgy;
+
+import sorg.testing._
+import util.parsing.input.CharArrayReader
+
+
+object ConfigLexerTests extends Tests {
+
+ override def testName = "ConfigLexerTests"
+
+ test("lexer1") {
+ expect("OpenTag(hello);EOF") {
+ (new ConfigLexer).scan("<hello>").mkString(";")
+ }
+
+ expect("Number(98.5);EOF") {
+ (new ConfigLexer).scan("98.5 ").mkString(";")
+ }
+
+ expect("QuotedString(\"hello there\");EOF") {
+ (new ConfigLexer).scan("\"hell\\x6F th\\x65re\"").mkString(";")
+ }
+ }
+
+ test("lexer2") {
+ expect("Ident(alpha);Assign;Number(-4);Ident(and);CondAssign;QuotedString(\"ok\");Assign;EOF") {
+ (new ConfigLexer).scan("alpha = -4 and ?= \"ok\" =").mkString(";")
+ }
+ }
+
+}
23 test/net/lag/configgy/StringUtilsTests.scala
@@ -0,0 +1,23 @@
+package net.lag.configgy
+
+import sorg.testing._
+
+
+object StringUtilsTests extends Tests {
+ override def testName = "StringUtilsTests"
+
+ test("quoteC") {
+ expect("nothing") { StringUtils.quoteC("nothing") }
+ expect("name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac") { StringUtils.quoteC("name\tvalue\t\u20acb\u00fcllet?\u20ac") }
+ expect("she said \\\"hello\\\"") { StringUtils.quoteC("she said \"hello\"") }
+ expect("\\\\backslash") { StringUtils.quoteC("\\backslash") }
+ }
+
+ test("unquoteC") {
+ expect("nothing") { StringUtils.unquoteC("nothing") }
+ expect("name\tvalue\t\u20acb\u00fcllet?\u20ac") { StringUtils.unquoteC("name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac") }
+ expect("she said \"hello\"") { StringUtils.unquoteC("she said \\\"hello\\\"") }
+ expect("\\backslash") { StringUtils.unquoteC("\\\\backslash") }
+ }
+
+}
35 test/net/lag/configgy/TestRunner.scala
@@ -0,0 +1,35 @@
+package net.lag.configgy;
+
+import scala.testing.SUnit._
+
+
+object TestRunner {
+ def sum(x: Seq[Int]) = (0 /: x) { _ + _ }
+
+ def main(args:Array[String]): Unit = {
+ val results = new TestResult
+
+ val suite = new TestSuite(StringUtilsTests,
+ ConfigLexerTests)
+
+ val testCount = sum(for (t <- suite.buf) yield t.asInstanceOf[sorg.testing.Tests].tests.length)
+ Console.println("Running " + testCount + " unit tests:")
+ suite.run(results)
+
+ if (! results.failures.hasNext) {
+ Console.println("Success!");
+ } else {
+ Console.println
+ Console println("FAILED TESTS (" + results.failureCount + "):");
+ Console.println
+ for (val each:TestFailure <- results.failures) {
+ Console.println(each.toString + ":")
+ for (val line <- each.trace.split("\n")) {
+ Console.println(" " + line)
+ }
+ Console.println
+ }
+ }
+ }
+
+}
56 test/sorg/testing/Tests.scala
@@ -0,0 +1,56 @@
+package sorg.testing;
+
+import scala.testing.SUnit._;
+
+abstract class Tests extends Test with Assert {
+ type TestExp = () => Unit;
+ var tests = List[Pair[String, TestExp]]();
+
+ def testName = "tests"
+
+ def test(desc: String)(t: => Unit) : Unit = {
+ // careful: this pushes the tests in reverse order like a stack, so
+ // we have to reverse them back later.
+ tests = Pair(desc, () => t) :: tests
+ }
+
+ override def run(tr: TestResult) = {
+ for (val Pair(desc, expression) <- tests.reverse) new TestCase(desc) {
+ override def runTest() = {
+ Console.print(" " + testName + ":" + desc + " ...")
+ Console.flush
+ try {
+ expression()
+ Console.println(" OK")
+ } catch {
+ case x => {
+ Console.println(" FAIL")
+ throw x
+ }
+ }
+ }
+ }.run(tr)
+ }
+
+ def expectThrow[T](throwClass: Class[T])(f: => Unit): Unit = {
+ try {
+ f
+ } catch {
+ case x => {
+ if (! throwClass.isAssignableFrom(x.getClass)) {
+ fail("Unexpected exception: " + x)
+ }
+ }
+ return
+ }
+ fail("Expected exception " + throwClass.getName)
+ }
+
+ def expect(expected: Any)(f: => Any): Unit = {
+ val actual = f
+ if (actual != expected) {
+ throw new AssertionError("expected '" + expected + "', got '" + actual + "'")
+ }
+ }
+
+}

0 comments on commit b29e1be

Please sign in to comment.