Skip to content

Commit

Permalink
Token
Browse files Browse the repository at this point in the history
  • Loading branch information
@xuwei-k
xuwei-k committed Apr 15, 2023
1 parent 03a4b27 commit 310b384
Show file tree
Hide file tree
Showing 5 changed files with 176 additions and 6 deletions.
86 changes: 81 additions & 5 deletions core/src/main/scala/scalameta_ast/ScalametaAST.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import scala.meta._
import scala.meta.common.Convert
import scala.meta.parsers.Parse
import scala.meta.parsers.Parsed
import scala.meta.tokens.Token
import scala.util.control.NonFatal

class ScalametaAST {
Expand Down Expand Up @@ -121,6 +122,72 @@ class ScalametaAST {
implicitly[Parse[Term]].apply(Input.String(str), dialects.Scala3).toOption.exists(_.is[Term.Name])
}

private def treeToString(tree: Tree, treeType: TreeType): String = {
treeType match {
case TreeType.Tree =>
scalametaBugWorkaround.foldLeft(tree.structure) { case (s, (x1, x2)) =>
s.replace(x1, x2)
}
case TreeType.Token =>
tree.tokens.tokens.map { x =>
val n = x.getClass.getSimpleName
def q(a: String): String = s"(\"${a}\")"

PartialFunction
.condOpt(x) {
case y: Token.Ident =>
s"Ident${q(y.value)}"
case y: Token.Comment =>
s"Comment${q(y.value)}"
case y: Token.Interpolation.Id =>
s"Interpolation.Id${q(y.value)}"
case y: Token.Interpolation.Part =>
s"Interpolation.Part${q(y.value)}"
case _: Token.Interpolation.Start =>
s"Interpolation.$n"
case _: Token.Interpolation.SpliceStart =>
s"Interpolation.$n"
case _: Token.Interpolation.SpliceEnd =>
s"Interpolation.$n"
case _: Token.Interpolation.End =>
s"Interpolation.$n"
case y: Token.Constant[?] =>
y match {
case z: Token.Constant.Int =>
s"Constant.Int(BigInt${q(z.value.toString)})"
case z: Token.Constant.Long =>
s"Constant.Long(BigInt${q(z.value.toString)})"
case z: Token.Constant.Float =>
s"Constant.Float(BigDecimal${q(z.value.toString)})"
case z: Token.Constant.Double =>
s"Constant.Double(BigDecimal${q(z.value.toString)})"
case z: Token.Constant.Char =>
s"Constant.Char('${z.value}')"
case z: Token.Constant.Symbol =>
s"Constant.Symbol(scala.Symbol${q(z.value.name)})"
case z: Token.Constant.String =>
s"Constant.String${q(z.value)}"
}
case _: Token.Xml.Start =>
s"Xml.${n}"
case y: Token.Xml.Part =>
s"Xml.Part${q(y.value)}"
case _: Token.Xml.SpliceStart =>
s"Xml.${n}"
case _: Token.Xml.SpliceEnd =>
s"Xml.${n}"
case _: Token.Xml.End =>
s"Xml.${n}"
case _: Token.Indentation.Indent =>
s"Indentation.${n}"
case _: Token.Indentation.Outdent =>
s"Indentation.${n}"
}
.getOrElse(n)
}.map("Token." + _).mkString("Seq(", ", ", ")")
}
}

def convert(
src: String,
format: Boolean,
Expand All @@ -134,7 +201,7 @@ class ScalametaAST {
): Output = {
val input = convert.apply(src)
val (ast, astBuildMs) = stopwatch {
val x = loop(
val tree = loop(
input,
for {
x1 <- parsers
Expand All @@ -147,10 +214,19 @@ class ScalametaAST {
)
}
} yield (x1, x2)
).structure
scalametaBugWorkaround.foldLeft(x) { case (s, (x1, x2)) =>
s.replace(x1, x2)
}
)

treeToString(
tree = tree,
treeType = {
outputType match {
case "tokens" =>
TreeType.Token
case _ =>
TreeType.Tree
}
}
)
}
val ruleNameRaw = ruleNameOption.getOrElse("Example").filterNot(char => char == '`' || char == '"')
val ruleName = {
Expand Down
8 changes: 8 additions & 0 deletions core/src/main/scala/scalameta_ast/TreeType.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package scalameta_ast

sealed abstract class TreeType extends Product with Serializable

object TreeType {
case object Tree extends TreeType
case object Token extends TreeType
}
79 changes: 79 additions & 0 deletions core/src/test/scala/scalameta_ast/ScalametaASTSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,85 @@ class ScalametaASTSpec extends AnyFreeSpec {
|""".stripMargin
assert(result.ast == expect)
}

"convert token" in {
val result = main.convert(
src = """def x(y: Z) = ('y, 'a', "b", 1.5, 4.4f, 2L, 3, s"x1${x2}", <g>{p}</g>) // c """,
format = true,
scalafmtConfig = metaconfig.Conf.Obj.empty,
outputType = "tokens",
packageName = Option("package_name"),
wildcardImport = false,
ruleNameOption = None,
dialect = Some("Scala213"),
patch = None,
)
val expect = """Seq(
| Token.BOF,
| Token.KwDef,
| Token.Space,
| Token.Ident("x"),
| Token.LeftParen,
| Token.Ident("y"),
| Token.Colon,
| Token.Space,
| Token.Ident("Z"),
| Token.RightParen,
| Token.Space,
| Token.Equals,
| Token.Space,
| Token.LeftParen,
| Token.Constant.Symbol(scala.Symbol("y")),
| Token.Comma,
| Token.Space,
| Token.Constant.Char('a'),
| Token.Comma,
| Token.Space,
| Token.Constant.String("b"),
| Token.Comma,
| Token.Space,
| Token.Constant.Double(BigDecimal("1.5")),
| Token.Comma,
| Token.Space,
| Token.Constant.Float(BigDecimal("4.4")),
| Token.Comma,
| Token.Space,
| Token.Constant.Long(BigInt("2")),
| Token.Comma,
| Token.Space,
| Token.Constant.Int(BigInt("3")),
| Token.Comma,
| Token.Space,
| Token.Interpolation.Id("s"),
| Token.Interpolation.Start,
| Token.Interpolation.Part("x1"),
| Token.Interpolation.SpliceStart,
| Token.LeftBrace,
| Token.Ident("x2"),
| Token.RightBrace,
| Token.Interpolation.SpliceEnd,
| Token.Interpolation.Part(""),
| Token.Interpolation.End,
| Token.Comma,
| Token.Space,
| Token.Xml.Start,
| Token.Xml.Part("<g>"),
| Token.Xml.SpliceStart,
| Token.LeftBrace,
| Token.Ident("p"),
| Token.RightBrace,
| Token.Xml.SpliceEnd,
| Token.Xml.Part("</g>"),
| Token.Xml.End,
| Token.RightParen,
| Token.Space,
| Token.Comment(" c "),
| Token.EOF
|)
|""".stripMargin
assert(result.ast == expect)
}

(0 to 1).foreach { i =>
s"top level scalameta classes ${i}" in {
val src = TestCompat.scalametaTreeFile(i)
Expand Down
4 changes: 4 additions & 0 deletions sources/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@
/>
<label for="semantic">Scalafix SemanticRule</label>
</div>
<div>
<input type="radio" id="tokens" name="output_type" value="tokens" />
<label for="tokens">Tokens</label>
</div>
</fieldset>
</div>
<div class="row">
Expand Down
5 changes: 4 additions & 1 deletion sources/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ $(() => {
const patch = $("#patch").val();

["package", "rule_name", "wildcard_import", "patch"].forEach((i) =>
$(`#${i}`).prop("disabled", outputType === "raw")
$(`#${i}`).prop("disabled", outputType === "raw" || outputType === "tokens")
);

const r = main.convert(
Expand Down Expand Up @@ -230,6 +230,9 @@ $(() => {
case "syntactic":
$("input[name=output_type][value='syntactic']").prop("checked", true);
break;
case "tokens":
$("input[name=output_type][value='tokens']").prop("checked", true);
break;
default:
$("input[name=output_type][value='raw']").prop("checked", true);
}
Expand Down

0 comments on commit 310b384

Please sign in to comment.