diff --git a/core/src/main/scala/scalameta_ast/ScalametaAST.scala b/core/src/main/scala/scalameta_ast/ScalametaAST.scala
index 1e1d2b1..3cf7467 100644
--- a/core/src/main/scala/scalameta_ast/ScalametaAST.scala
+++ b/core/src/main/scala/scalameta_ast/ScalametaAST.scala
@@ -8,6 +8,7 @@ import scala.meta._
import scala.meta.common.Convert
import scala.meta.parsers.Parse
import scala.meta.parsers.Parsed
+import scala.meta.tokens.Token
import scala.util.control.NonFatal
class ScalametaAST {
@@ -121,6 +122,72 @@ class ScalametaAST {
implicitly[Parse[Term]].apply(Input.String(str), dialects.Scala3).toOption.exists(_.is[Term.Name])
}
+ private def treeToString(tree: Tree, treeType: TreeType): String = {
+ treeType match {
+ case TreeType.Tree =>
+ scalametaBugWorkaround.foldLeft(tree.structure) { case (s, (x1, x2)) =>
+ s.replace(x1, x2)
+ }
+ case TreeType.Token =>
+ tree.tokens.tokens.map { x =>
+ val n = x.getClass.getSimpleName
+ def q(a: String): String = s"(\"${a}\")"
+
+ PartialFunction
+ .condOpt(x) {
+ case y: Token.Ident =>
+ s"Ident${q(y.value)}"
+ case y: Token.Comment =>
+ s"Comment${q(y.value)}"
+ case y: Token.Interpolation.Id =>
+ s"Interpolation.Id${q(y.value)}"
+ case y: Token.Interpolation.Part =>
+ s"Interpolation.Part${q(y.value)}"
+ case _: Token.Interpolation.Start =>
+ s"Interpolation.$n"
+ case _: Token.Interpolation.SpliceStart =>
+ s"Interpolation.$n"
+ case _: Token.Interpolation.SpliceEnd =>
+ s"Interpolation.$n"
+ case _: Token.Interpolation.End =>
+ s"Interpolation.$n"
+ case y: Token.Constant[?] =>
+ y match {
+ case z: Token.Constant.Int =>
+ s"Constant.Int(BigInt${q(z.value.toString)})"
+ case z: Token.Constant.Long =>
+ s"Constant.Long(BigInt${q(z.value.toString)})"
+ case z: Token.Constant.Float =>
+ s"Constant.Float(BigDecimal${q(z.value.toString)})"
+ case z: Token.Constant.Double =>
+ s"Constant.Double(BigDecimal${q(z.value.toString)})"
+ case z: Token.Constant.Char =>
+ s"Constant.Char('${z.value}')"
+ case z: Token.Constant.Symbol =>
+ s"Constant.Symbol(scala.Symbol${q(z.value.name)})"
+ case z: Token.Constant.String =>
+ s"Constant.String${q(z.value)}"
+ }
+ case _: Token.Xml.Start =>
+ s"Xml.${n}"
+ case y: Token.Xml.Part =>
+ s"Xml.Part${q(y.value)}"
+ case _: Token.Xml.SpliceStart =>
+ s"Xml.${n}"
+ case _: Token.Xml.SpliceEnd =>
+ s"Xml.${n}"
+ case _: Token.Xml.End =>
+ s"Xml.${n}"
+ case _: Token.Indentation.Indent =>
+ s"Indentation.${n}"
+ case _: Token.Indentation.Outdent =>
+ s"Indentation.${n}"
+ }
+ .getOrElse(n)
+ }.map("Token." + _).mkString("Seq(", ", ", ")")
+ }
+ }
+
def convert(
src: String,
format: Boolean,
@@ -134,7 +201,7 @@ class ScalametaAST {
): Output = {
val input = convert.apply(src)
val (ast, astBuildMs) = stopwatch {
- val x = loop(
+ val tree = loop(
input,
for {
x1 <- parsers
@@ -147,10 +214,19 @@ class ScalametaAST {
)
}
} yield (x1, x2)
- ).structure
- scalametaBugWorkaround.foldLeft(x) { case (s, (x1, x2)) =>
- s.replace(x1, x2)
- }
+ )
+
+ treeToString(
+ tree = tree,
+ treeType = {
+ outputType match {
+ case "tokens" =>
+ TreeType.Token
+ case _ =>
+ TreeType.Tree
+ }
+ }
+ )
}
val ruleNameRaw = ruleNameOption.getOrElse("Example").filterNot(char => char == '`' || char == '"')
val ruleName = {
diff --git a/core/src/main/scala/scalameta_ast/TreeType.scala b/core/src/main/scala/scalameta_ast/TreeType.scala
new file mode 100644
index 0000000..1f3d855
--- /dev/null
+++ b/core/src/main/scala/scalameta_ast/TreeType.scala
@@ -0,0 +1,8 @@
+package scalameta_ast
+
+sealed abstract class TreeType extends Product with Serializable
+
+object TreeType {
+ case object Tree extends TreeType
+ case object Token extends TreeType
+}
diff --git a/core/src/test/scala/scalameta_ast/ScalametaASTSpec.scala b/core/src/test/scala/scalameta_ast/ScalametaASTSpec.scala
index bc02e7b..8679626 100644
--- a/core/src/test/scala/scalameta_ast/ScalametaASTSpec.scala
+++ b/core/src/test/scala/scalameta_ast/ScalametaASTSpec.scala
@@ -58,6 +58,85 @@ class ScalametaASTSpec extends AnyFreeSpec {
|""".stripMargin
assert(result.ast == expect)
}
+
+ "convert token" in {
+ val result = main.convert(
+ src = """def x(y: Z) = ('y, 'a', "b", 1.5, 4.4f, 2L, 3, s"x1${x2}", {p}) // c """,
+ format = true,
+ scalafmtConfig = metaconfig.Conf.Obj.empty,
+ outputType = "tokens",
+ packageName = Option("package_name"),
+ wildcardImport = false,
+ ruleNameOption = None,
+ dialect = Some("Scala213"),
+ patch = None,
+ )
+ val expect = """Seq(
+ | Token.BOF,
+ | Token.KwDef,
+ | Token.Space,
+ | Token.Ident("x"),
+ | Token.LeftParen,
+ | Token.Ident("y"),
+ | Token.Colon,
+ | Token.Space,
+ | Token.Ident("Z"),
+ | Token.RightParen,
+ | Token.Space,
+ | Token.Equals,
+ | Token.Space,
+ | Token.LeftParen,
+ | Token.Constant.Symbol(scala.Symbol("y")),
+ | Token.Comma,
+ | Token.Space,
+ | Token.Constant.Char('a'),
+ | Token.Comma,
+ | Token.Space,
+ | Token.Constant.String("b"),
+ | Token.Comma,
+ | Token.Space,
+ | Token.Constant.Double(BigDecimal("1.5")),
+ | Token.Comma,
+ | Token.Space,
+ | Token.Constant.Float(BigDecimal("4.4")),
+ | Token.Comma,
+ | Token.Space,
+ | Token.Constant.Long(BigInt("2")),
+ | Token.Comma,
+ | Token.Space,
+ | Token.Constant.Int(BigInt("3")),
+ | Token.Comma,
+ | Token.Space,
+ | Token.Interpolation.Id("s"),
+ | Token.Interpolation.Start,
+ | Token.Interpolation.Part("x1"),
+ | Token.Interpolation.SpliceStart,
+ | Token.LeftBrace,
+ | Token.Ident("x2"),
+ | Token.RightBrace,
+ | Token.Interpolation.SpliceEnd,
+ | Token.Interpolation.Part(""),
+ | Token.Interpolation.End,
+ | Token.Comma,
+ | Token.Space,
+ | Token.Xml.Start,
+ | Token.Xml.Part(""),
+ | Token.Xml.SpliceStart,
+ | Token.LeftBrace,
+ | Token.Ident("p"),
+ | Token.RightBrace,
+ | Token.Xml.SpliceEnd,
+ | Token.Xml.Part(""),
+ | Token.Xml.End,
+ | Token.RightParen,
+ | Token.Space,
+ | Token.Comment(" c "),
+ | Token.EOF
+ |)
+ |""".stripMargin
+ assert(result.ast == expect)
+ }
+
(0 to 1).foreach { i =>
s"top level scalameta classes ${i}" in {
val src = TestCompat.scalametaTreeFile(i)
diff --git a/sources/index.html b/sources/index.html
index 32ad9cd..2c009ef 100644
--- a/sources/index.html
+++ b/sources/index.html
@@ -104,6 +104,10 @@
/>
+
+
+
+
diff --git a/sources/main.js b/sources/main.js
index 4aa6c0c..06b4fdd 100644
--- a/sources/main.js
+++ b/sources/main.js
@@ -55,7 +55,7 @@ $(() => {
const patch = $("#patch").val();
["package", "rule_name", "wildcard_import", "patch"].forEach((i) =>
- $(`#${i}`).prop("disabled", outputType === "raw")
+ $(`#${i}`).prop("disabled", outputType === "raw" || outputType === "tokens")
);
const r = main.convert(
@@ -230,6 +230,9 @@ $(() => {
case "syntactic":
$("input[name=output_type][value='syntactic']").prop("checked", true);
break;
+ case "tokens":
+ $("input[name=output_type][value='tokens']").prop("checked", true);
+ break;
default:
$("input[name=output_type][value='raw']").prop("checked", true);
}