diff --git a/.changeset/yellow-numbers-leave.md b/.changeset/yellow-numbers-leave.md
new file mode 100644
index 000000000..3079a7bc3
--- /dev/null
+++ b/.changeset/yellow-numbers-leave.md
@@ -0,0 +1,5 @@
+---
+'@astrojs/compiler': minor
+---
+
+Adds support for `Astro.self` (as accepted in the [Recursive Components RFC](https://github.com/withastro/rfcs/blob/main/active-rfcs/0000-recursive-components.md)).
diff --git a/cmd/astro-wasm/astro-wasm.go b/cmd/astro-wasm/astro-wasm.go
index 94d5a64eb..a77cb6dbd 100644
--- a/cmd/astro-wasm/astro-wasm.go
+++ b/cmd/astro-wasm/astro-wasm.go
@@ -14,6 +14,7 @@ import (
"github.com/norunners/vert"
astro "github.com/withastro/compiler/internal"
"github.com/withastro/compiler/internal/printer"
+ t "github.com/withastro/compiler/internal/t"
"github.com/withastro/compiler/internal/transform"
wasm_utils "github.com/withastro/compiler/internal_wasm/utils"
"golang.org/x/net/html/atom"
@@ -23,6 +24,7 @@ var done chan bool
func main() {
js.Global().Set("__astro_transform", Transform())
+ js.Global().Set("__astro_parse", Parse())
// This ensures that the WASM doesn't exit early
<-make(chan bool)
}
@@ -41,6 +43,19 @@ func jsBool(j js.Value) bool {
return j.Bool()
}
+func makeParseOptions(options js.Value) t.ParseOptions {
+ position := true
+
+ pos := options.Get("position")
+ if !pos.IsNull() && !pos.IsUndefined() {
+ position = pos.Bool()
+ }
+
+ return t.ParseOptions{
+ Position: position,
+ }
+}
+
func makeTransformOptions(options js.Value, hash string) transform.TransformOptions {
filename := jsString(options.Get("sourcefile"))
if filename == "" {
@@ -107,6 +122,10 @@ type HoistedScript struct {
Type string `js:"type"`
}
+type ParseResult struct {
+ AST string `js:"ast"`
+}
+
type TransformResult struct {
Code string `js:"code"`
Map string `js:"map"`
@@ -133,6 +152,36 @@ func preprocessStyle(i int, style *astro.Node, transformOptions transform.Transf
style.FirstChild.Data = str
}
+func Parse() interface{} {
+ return js.FuncOf(func(this js.Value, args []js.Value) interface{} {
+ source := jsString(args[0])
+ parseOptions := makeParseOptions(js.Value(args[1]))
+
+ var doc *astro.Node
+ nodes, err := astro.ParseFragment(strings.NewReader(source), &astro.Node{
+ Type: astro.ElementNode,
+ Data: atom.Template.String(),
+ DataAtom: atom.Template,
+ })
+ if err != nil {
+ fmt.Println(err)
+ }
+ doc = &astro.Node{
+ Type: astro.DocumentNode,
+ }
+ for i := 0; i < len(nodes); i++ {
+ n := nodes[i]
+ doc.AppendChild(n)
+ }
+
+ result := printer.PrintToJSON(source, doc, parseOptions)
+
+ return vert.ValueOf(ParseResult{
+ AST: string(result.Output),
+ })
+ })
+}
+
func Transform() interface{} {
return js.FuncOf(func(this js.Value, args []js.Value) interface{} {
source := jsString(args[0])
diff --git a/go.mod b/go.mod
index 97342ec73..00a259e1b 100644
--- a/go.mod
+++ b/go.mod
@@ -11,4 +11,6 @@ require (
golang.org/x/net v0.0.0-20210716203947-853a461950ff
)
+require github.com/iancoleman/strcase v0.2.0 // indirect
+
replace github.com/norunners/vert => github.com/natemoo-re/vert v0.0.0-natemoo-re.7
diff --git a/go.sum b/go.sum
index ac607f966..4530539b0 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,7 @@
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
+github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
github.com/natemoo-re/vert v0.0.0-natemoo-re.7 h1:nhfKslS16o2Uruqt8Bwv8ZFYUuf+PW9iC2M5HI/Bs6U=
diff --git a/internal/const.go b/internal/const.go
index 3aad85779..370a3185d 100644
--- a/internal/const.go
+++ b/internal/const.go
@@ -4,6 +4,8 @@
package astro
+import a "golang.org/x/net/html/atom"
+
// Section 12.2.4.2 of the HTML5 specification says "The following elements
// have varying levels of special parsing rules".
// https://html.spec.whatwg.org/multipage/syntax.html#the-stack-of-open-elements
@@ -109,3 +111,26 @@ func isSpecialElement(element *Node) bool {
}
return false
}
+
+var knownDirectiveMap = map[string]bool{
+ "client:load": true,
+ "client:idle": true,
+ "client:visible": true,
+ "client:only": true,
+ "class:list": true,
+ "set:text": true,
+ "set:html": true,
+}
+
+func IsKnownDirective(element *Node, attr *Attribute) bool {
+ if knownDirectiveMap[attr.Key] {
+ return true
+ }
+ if element.DataAtom == a.Script {
+ return attr.Key == "hoist"
+ }
+ if element.DataAtom == a.Style {
+ return attr.Key == "global"
+ }
+ return false
+}
diff --git a/internal/node.go b/internal/node.go
index 0c55b5cab..3e920f117 100644
--- a/internal/node.go
+++ b/internal/node.go
@@ -30,6 +30,29 @@ const (
ExpressionNode
)
+func (t NodeType) String() string {
+ switch t {
+ case ErrorNode:
+ return "error"
+ case TextNode:
+ return "text"
+ case DocumentNode:
+ return "root"
+ case ElementNode:
+ return "element"
+ case CommentNode:
+ return "comment"
+ case DoctypeNode:
+ return "doctype"
+ case FrontmatterNode:
+ return "frontmatter"
+ case ExpressionNode:
+ return "expression"
+ default:
+ return ""
+ }
+}
+
// Used as an Attribute Key to mark implicit nodes
const ImplicitNodeMarker = "\x00implicit"
diff --git a/internal/parser.go b/internal/parser.go
index 39a868d49..88b6c5827 100644
--- a/internal/parser.go
+++ b/internal/parser.go
@@ -365,6 +365,7 @@ func (p *parser) addFrontmatter(empty bool) {
}
if empty {
p.frontmatterState = FrontmatterClosed
+ p.fm.Attr = append(p.fm.Attr, Attribute{Key: ImplicitNodeMarker, Type: EmptyAttribute})
} else {
p.frontmatterState = FrontmatterOpen
p.oe = append(p.oe, p.fm)
diff --git a/internal/printer/print-to-js.go b/internal/printer/print-to-js.go
index 751195304..a2352d8aa 100644
--- a/internal/printer/print-to-js.go
+++ b/internal/printer/print-to-js.go
@@ -106,8 +106,7 @@ func render1(p *printer, n *Node, opts RenderOptions) {
}
p.printReturnClose()
- // TODO: use proper component name
- p.printFuncSuffix("$$Component")
+ p.printFuncSuffix(opts.opts)
return
}
@@ -149,8 +148,7 @@ func render1(p *printer, n *Node, opts RenderOptions) {
// 3. The metadata object
p.printComponentMetadata(n.Parent, opts.opts, []byte(c.Data))
- // TODO: use the proper component name
- p.printFuncPrelude("$$Component")
+ p.printFuncPrelude(opts.opts)
} else {
importStatements := c.Data[0:renderBodyStart]
content := c.Data[renderBodyStart:]
@@ -176,8 +174,7 @@ func render1(p *printer, n *Node, opts RenderOptions) {
}
}
- // TODO: use the proper component name
- p.printFuncPrelude("$$Component")
+ p.printFuncPrelude(opts.opts)
if len(c.Loc) > 0 {
p.addSourceMapping(loc.Loc{Start: c.Loc[0].Start + renderBodyStart})
}
@@ -225,8 +222,7 @@ func render1(p *printer, n *Node, opts RenderOptions) {
p.printTopLevelAstro(opts.opts)
// Render func prelude. Will only run for the first non-frontmatter node
- // TODO: use the proper component name
- p.printFuncPrelude("$$Component")
+ p.printFuncPrelude(opts.opts)
// This just ensures a newline
p.println("")
diff --git a/internal/printer/print-to-json.go b/internal/printer/print-to-json.go
new file mode 100644
index 000000000..6780aef57
--- /dev/null
+++ b/internal/printer/print-to-json.go
@@ -0,0 +1,241 @@
+package printer
+
+import (
+ "fmt"
+ "regexp"
+ "strings"
+
+ . "github.com/withastro/compiler/internal"
+ "github.com/withastro/compiler/internal/loc"
+ "github.com/withastro/compiler/internal/sourcemap"
+ "github.com/withastro/compiler/internal/t"
+ "github.com/withastro/compiler/internal/transform"
+)
+
+type ASTPosition struct {
+ Start ASTPoint `json:"start,omitempty"`
+ End ASTPoint `json:"end,omitempty"`
+}
+
+type ASTPoint struct {
+ Line int `json:"line,omitempty"`
+ Column int `json:"column,omitempty"`
+ Offset int `json:"offset,omitempty"`
+}
+
+type ASTNode struct {
+ Type string `json:"type"`
+ Name string `json:"name,omitempty"`
+ Value string `json:"value,omitempty"`
+ Attributes []ASTNode `json:"attributes,omitempty"`
+ Directives []ASTNode `json:"directives,omitempty"`
+ Children []ASTNode `json:"children,omitempty"`
+ Position ASTPosition `json:"position,omitempty"`
+
+ // Attributes only
+ Kind string `json:"kind,omitempty"`
+}
+
+func escapeForJSON(value string) string {
+ newlines := regexp.MustCompile(`\n`)
+ value = newlines.ReplaceAllString(value, `\n`)
+ doublequotes := regexp.MustCompile(`"`)
+ value = doublequotes.ReplaceAllString(value, `\"`)
+ amp := regexp.MustCompile(`&`)
+ value = amp.ReplaceAllString(value, `\&`)
+ r := regexp.MustCompile(`\r`)
+ value = r.ReplaceAllString(value, `\r`)
+ t := regexp.MustCompile(`\t`)
+ value = t.ReplaceAllString(value, `\t`)
+ f := regexp.MustCompile(`\f`)
+ value = f.ReplaceAllString(value, `\f`)
+ return value
+}
+
+func (n ASTNode) String() string {
+ str := fmt.Sprintf(`{"type":"%s"`, n.Type)
+ if n.Kind != "" {
+ str += fmt.Sprintf(`,"kind":"%s"`, n.Kind)
+ }
+ if n.Name != "" {
+ str += fmt.Sprintf(`,"name":"%s"`, escapeForJSON(n.Name))
+ }
+ if n.Value != "" || n.Type == "attribute" {
+ str += fmt.Sprintf(`,"value":"%s"`, escapeForJSON(n.Value))
+ }
+ if len(n.Attributes) > 0 {
+ str += `,"attributes":[`
+ for i, attr := range n.Attributes {
+ str += attr.String()
+ if i < len(n.Attributes)-1 {
+ str += ","
+ }
+ }
+ str += `]`
+ }
+ if len(n.Directives) > 0 {
+ str += `,"directives":[`
+ for i, attr := range n.Directives {
+ str += attr.String()
+ if i < len(n.Directives)-1 {
+ str += ","
+ }
+ }
+ str += `]`
+ }
+ if len(n.Children) > 0 {
+ str += `,"children":[`
+ for i, node := range n.Children {
+ str += node.String()
+ if i < len(n.Children)-1 {
+ str += ","
+ }
+ }
+ str += `]`
+ }
+ if n.Position.Start.Line != 0 {
+ str += `,"position":{`
+ str += fmt.Sprintf(`"start":{"line":%d,"column":%d,"offset":%d}`, n.Position.Start.Line, n.Position.Start.Column, n.Position.Start.Offset)
+ if n.Position.End.Line != 0 {
+ str += fmt.Sprintf(`,"end":{"line":%d,"column":%d,"offset":%d}`, n.Position.End.Line, n.Position.End.Column, n.Position.End.Offset)
+ }
+ str += "}"
+ }
+ str += "}"
+ return str
+}
+
+func PrintToJSON(sourcetext string, n *Node, opts t.ParseOptions) PrintResult {
+ p := &printer{
+ builder: sourcemap.MakeChunkBuilder(nil, sourcemap.GenerateLineOffsetTables(sourcetext, len(strings.Split(sourcetext, "\n")))),
+ }
+ root := ASTNode{}
+ renderNode(p, &root, n, opts)
+ doc := root.Children[0]
+ return PrintResult{
+ Output: []byte(doc.String()),
+ }
+}
+
+func locToPoint(p *printer, loc loc.Loc) ASTPoint {
+ offset := loc.Start
+ info := p.builder.GetLineAndColumnForLocation(loc)
+ line := info[0]
+ column := info[1]
+
+ return ASTPoint{
+ Line: line,
+ Column: column,
+ Offset: offset,
+ }
+}
+
+func positionAt(p *printer, n *Node, opts t.ParseOptions) ASTPosition {
+ if !opts.Position {
+ return ASTPosition{}
+ }
+
+ if len(n.Loc) == 1 {
+ s := n.Loc[0]
+ start := locToPoint(p, s)
+
+ return ASTPosition{
+ Start: start,
+ }
+ }
+
+ if len(n.Loc) == 2 {
+ s := n.Loc[0]
+ e := n.Loc[1]
+ start := locToPoint(p, s)
+ end := locToPoint(p, e)
+
+ return ASTPosition{
+ Start: start,
+ End: end,
+ }
+ }
+ return ASTPosition{}
+}
+
+func attrPositionAt(p *printer, n *Attribute, opts t.ParseOptions) ASTPosition {
+ if !opts.Position {
+ return ASTPosition{}
+ }
+
+ k := n.KeyLoc
+ start := locToPoint(p, k)
+
+ return ASTPosition{
+ Start: start,
+ }
+}
+
+func renderNode(p *printer, parent *ASTNode, n *Node, opts t.ParseOptions) {
+ isImplicit := false
+ for _, a := range n.Attr {
+ if transform.IsImplictNodeMarker(a) {
+ isImplicit = true
+ break
+ }
+ }
+ hasChildren := n.FirstChild != nil
+ if isImplicit {
+ if hasChildren {
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ renderNode(p, parent, c, opts)
+ }
+ }
+ return
+ }
+ var node ASTNode
+
+ node.Position = positionAt(p, n, opts)
+
+ if n.Type == ElementNode {
+ if n.Expression {
+ node.Type = "expression"
+ } else {
+ node.Name = n.Data
+ if n.Component {
+ node.Type = "component"
+ } else if n.CustomElement {
+ node.Type = "custom-element"
+ } else {
+ node.Type = "element"
+ }
+
+ for _, attr := range n.Attr {
+ attrNode := ASTNode{
+ Type: "attribute",
+ Kind: attr.Type.String(),
+ Position: attrPositionAt(p, &attr, opts),
+ Name: attr.Key,
+ Value: attr.Val,
+ }
+ if IsKnownDirective(n, &attr) {
+ attrNode.Type = "directive"
+ node.Directives = append(node.Directives, attrNode)
+ } else {
+ node.Attributes = append(node.Attributes, attrNode)
+ }
+ }
+ }
+ } else {
+ node.Type = n.Type.String()
+ if n.Type == TextNode || n.Type == CommentNode || n.Type == DoctypeNode {
+ node.Value = n.Data
+ }
+ }
+ if n.Type == FrontmatterNode && hasChildren {
+ node.Value = n.FirstChild.Data
+ } else {
+ if !isImplicit && hasChildren {
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ renderNode(p, &node, c, opts)
+ }
+ }
+ }
+
+ parent.Children = append(parent.Children, node)
+}
diff --git a/internal/printer/printer.go b/internal/printer/printer.go
index b408077dc..4621f5aca 100644
--- a/internal/printer/printer.go
+++ b/internal/printer/printer.go
@@ -149,18 +149,21 @@ func (p *printer) printDefineVars(n *astro.Node) {
}
}
-func (p *printer) printFuncPrelude(componentName string) {
+func (p *printer) printFuncPrelude(opts transform.TransformOptions) {
if p.hasFuncPrelude {
return
}
+ componentName := getComponentName(opts.Pathname)
p.addNilSourceMapping()
p.println("\n//@ts-ignore")
p.println(fmt.Sprintf("const %s = %s(async (%s, $$props, %s) => {", componentName, CREATE_COMPONENT, RESULT, SLOTS))
p.println(fmt.Sprintf("const Astro = %s.createAstro($$Astro, $$props, %s);", RESULT, SLOTS))
+ p.println(fmt.Sprintf("Astro.self = %s;", componentName))
p.hasFuncPrelude = true
}
-func (p *printer) printFuncSuffix(componentName string) {
+func (p *printer) printFuncSuffix(opts transform.TransformOptions) {
+ componentName := getComponentName(opts.Pathname)
p.addNilSourceMapping()
p.println("});")
p.println(fmt.Sprintf("export default %s;", componentName))
diff --git a/internal/printer/printer_test.go b/internal/printer/printer_test.go
index 4718ab98a..27ac2b7d5 100644
--- a/internal/printer/printer_test.go
+++ b/internal/printer/printer_test.go
@@ -9,6 +9,7 @@ import (
"testing"
astro "github.com/withastro/compiler/internal"
+ types "github.com/withastro/compiler/internal/t"
"github.com/withastro/compiler/internal/test_utils"
"github.com/withastro/compiler/internal/transform"
)
@@ -29,7 +30,8 @@ var INTERNAL_IMPORTS = fmt.Sprintf("import {\n %s\n} from \"%s\";\n", strings.J
}, ",\n "), "http://localhost:3000/")
var PRELUDE = fmt.Sprintf(`//@ts-ignore
const $$Component = %s(async ($$result, $$props, %s) => {
-const Astro = $$result.createAstro($$Astro, $$props, %s);%s`, CREATE_COMPONENT, SLOTS, SLOTS, "\n")
+const Astro = $$result.createAstro($$Astro, $$props, %s);
+Astro.self = $$Component;%s`, CREATE_COMPONENT, SLOTS, SLOTS, "\n")
var RETURN = fmt.Sprintf("return %s%s", TEMPLATE_TAG, BACKTICK)
var SUFFIX = fmt.Sprintf("%s;", BACKTICK) + `
});
@@ -68,6 +70,12 @@ type testcase struct {
want want
}
+type jsonTestcase struct {
+ name string
+ source string
+ want []ASTNode
+}
+
func TestPrinter(t *testing.T) {
longRandomString := ""
for i := 0; i < 4080; i++ {
@@ -1610,3 +1618,73 @@ const items = ["Dog", "Cat", "Platipus"];
})
}
}
+
+func TestPrintToJSON(t *testing.T) {
+ tests := []jsonTestcase{
+ {
+ name: "basic",
+ source: `Hello world!
`,
+ want: []ASTNode{{Type: "element", Name: "h1", Children: []ASTNode{{Type: "text", Value: "Hello world!"}}}},
+ },
+ {
+ name: "expression",
+ source: `Hello {world}
`,
+ want: []ASTNode{{Type: "element", Name: "h1", Children: []ASTNode{{Type: "text", Value: "Hello "}, {Type: "expression", Children: []ASTNode{{Type: "text", Value: "world"}}}}}},
+ },
+ {
+ name: "Component",
+ source: ``,
+ want: []ASTNode{{Type: "component", Name: "Component"}},
+ },
+ {
+ name: "custom-element",
+ source: ``,
+ want: []ASTNode{{Type: "custom-element", Name: "custom-element"}},
+ },
+ {
+ name: "Doctype",
+ source: ``,
+ want: []ASTNode{{Type: "doctype", Value: "html"}},
+ },
+ {
+ name: "Comment",
+ source: ``,
+ want: []ASTNode{{Type: "comment", Value: "hello"}},
+ },
+ {
+ name: "Comment preserves whitespace",
+ source: ``,
+ want: []ASTNode{{Type: "comment", Value: " hello "}},
+ },
+ {
+ name: "Frontmatter",
+ source: `---
+const a = "hey"
+---
+{a}
`,
+ want: []ASTNode{{Type: "frontmatter", Value: "\nconst a = \"hey\"\n"}, {Type: "element", Name: "div", Children: []ASTNode{{Type: "expression", Children: []ASTNode{{Type: "text", Value: "a"}}}}}},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // transform output from source
+ code := test_utils.Dedent(tt.source)
+
+ doc, err := astro.Parse(strings.NewReader(code))
+
+ if err != nil {
+ t.Error(err)
+ }
+
+ root := ASTNode{Type: "root", Children: tt.want}
+ toMatch := root.String()
+
+ result := PrintToJSON(code, doc, types.ParseOptions{Position: false})
+
+ if diff := test_utils.ANSIDiff(test_utils.Dedent(string(toMatch)), test_utils.Dedent(string(result.Output))); diff != "" {
+ t.Error(fmt.Sprintf("mismatch (-want +got):\n%s", diff))
+ }
+ })
+ }
+}
diff --git a/internal/printer/utils.go b/internal/printer/utils.go
index c1304c033..6c4c05d30 100644
--- a/internal/printer/utils.go
+++ b/internal/printer/utils.go
@@ -3,6 +3,8 @@ package printer
import (
"regexp"
"strings"
+
+ "github.com/iancoleman/strcase"
)
func escapeText(src string) string {
@@ -13,6 +15,22 @@ func escapeText(src string) string {
)
}
+func getComponentName(pathname string) string {
+ if len(pathname) == 0 {
+ return "$$Component"
+ }
+ parts := strings.Split(pathname, "/")
+ part := parts[len(parts)-1]
+ if len(part) == 0 {
+ return "$$Component"
+ }
+ basename := strcase.ToCamel(strings.Split(part, ".")[0])
+ if basename == "Astro" {
+ return "$$Component"
+ }
+ return strings.Join([]string{"$$", basename}, "")
+}
+
func escapeExistingEscapes(src string) string {
return strings.Replace(src, "\\", "\\\\", -1)
}
diff --git a/internal/sourcemap/sourcemap.go b/internal/sourcemap/sourcemap.go
index 69425e5a1..b4250ccb1 100644
--- a/internal/sourcemap/sourcemap.go
+++ b/internal/sourcemap/sourcemap.go
@@ -607,6 +607,36 @@ func MakeChunkBuilder(inputSourceMap *SourceMap, lineOffsetTables []LineOffsetTa
}
}
+func (b *ChunkBuilder) GetLineAndColumnForLocation(location loc.Loc) []int {
+ b.prevLoc = location
+
+ // Binary search to find the line
+ lineOffsetTables := b.lineOffsetTables
+ count := len(lineOffsetTables)
+ originalLine := 0
+ for count > 0 {
+ step := count / 2
+ i := originalLine + step
+ if lineOffsetTables[i].byteOffsetToStartOfLine <= location.Start {
+ originalLine = i + 1
+ count = count - step - 1
+ } else {
+ count = step
+ }
+ }
+ originalLine--
+
+ // Use the line to compute the column
+ line := &lineOffsetTables[originalLine]
+ originalColumn := int(location.Start - line.byteOffsetToStartOfLine)
+ if line.columnsForNonASCII != nil && originalColumn >= int(line.byteOffsetToFirstNonASCII) {
+ originalColumn = int(line.columnsForNonASCII[originalColumn-int(line.byteOffsetToFirstNonASCII)])
+ }
+
+ // 1-based line, 1-based column
+ return []int{originalLine + 1, originalColumn + 1}
+}
+
func (b *ChunkBuilder) AddSourceMapping(location loc.Loc, output []byte) {
if location == b.prevLoc {
return
diff --git a/internal/t/t.go b/internal/t/t.go
new file mode 100644
index 000000000..f81714daa
--- /dev/null
+++ b/internal/t/t.go
@@ -0,0 +1,5 @@
+package t
+
+type ParseOptions struct {
+ Position bool
+}
diff --git a/internal/token.go b/internal/token.go
index 19687dd7b..c019dd278 100644
--- a/internal/token.go
+++ b/internal/token.go
@@ -67,6 +67,24 @@ const (
// AttributeType is the type of an Attribute
type AttributeType uint32
+func (t AttributeType) String() string {
+ switch t {
+ case QuotedAttribute:
+ return "quoted"
+ case EmptyAttribute:
+ return "empty"
+ case ExpressionAttribute:
+ return "expression"
+ case SpreadAttribute:
+ return "spread"
+ case ShorthandAttribute:
+ return "shorthand"
+ case TemplateLiteralAttribute:
+ return "template-literal"
+ }
+ return "Invalid(" + strconv.Itoa(int(t)) + ")"
+}
+
const (
QuotedAttribute AttributeType = iota
EmptyAttribute
@@ -1254,7 +1272,7 @@ func (z *Tokenizer) readTagAttrExpression() {
}
func (z *Tokenizer) Loc() loc.Loc {
- return loc.Loc{Start: z.raw.Start}
+ return loc.Loc{Start: z.data.Start}
}
// An expression boundary means the next tokens should be treated as a JS expression
diff --git a/lib/compiler/README.md b/lib/compiler/README.md
index 74c88ef2f..dedca63ef 100644
--- a/lib/compiler/README.md
+++ b/lib/compiler/README.md
@@ -12,7 +12,15 @@ npm install @astrojs/compiler
## Usage
-_Note: API will change before 1.0! Use at your own discretion._
+_Note: Public APIs are likely to change before 1.0! Use at your own discretion._
+
+#### Transform `.astro` to valid TypeScript
+
+The Astro compiler can convert `.astro` syntax to a TypeScript Module whose default export generates HTML.
+
+**Some notes**...
+- TypeScript is valid `.astro` syntax! The output code may need an additional post-processing step to generate valid JavaScript.
+- `.astro` files rely on a server implementation exposed as `astro/internal` in the Node ecosystem. Other runtimes currently need to bring their own rendering implementation and reference it via `internalURL`. This is a pain point we're looking into fixing.
```js
import { transform } from '@astrojs/compiler';
@@ -25,6 +33,31 @@ const result = await transform(source, {
});
```
+#### Parse `.astro` and return an AST
+
+The Astro compiler can emit an AST using the `parse` method.
+
+**Some notes**...
+- Position data is currently incomplete and in some cases incorrect. We're working on it!
+- A `TextNode` can represent both HTML `text` and JavaScript/TypeScript source code.
+- The `@astrojs/compiler/utils` entrypoint exposes a `walk` function that can be used to traverse the AST. It also exposes the `is` helper which can be used as guards to derive the proper types for each `node`.
+
+```js
+import { parse } from '@astrojs/compiler';
+import { walk, is } from '@astrojs/compiler/utils';
+
+const result = await parse(source, {
+ position: false, // defaults to `true`
+});
+
+walk(result.ast, (node) => {
+ // `tag` nodes are `element` | `custom-element` | `component`
+ if (is.tag(node)) {
+ console.log(node.name);
+ }
+})
+```
+
## Contributing
[CONTRIBUTING.md](./CONTRIBUTING.md)
diff --git a/lib/compiler/browser/index.ts b/lib/compiler/browser/index.ts
index ef2c9acb2..9b4d1ea9b 100644
--- a/lib/compiler/browser/index.ts
+++ b/lib/compiler/browser/index.ts
@@ -5,8 +5,13 @@ export const transform: typeof types.transform = (input, options) => {
return ensureServiceIsRunning().transform(input, options);
};
+export const parse: typeof types.parse = (input, options) => {
+ return ensureServiceIsRunning().parse(input, options);
+};
+
interface Service {
transform: typeof types.transform;
+ parse: typeof types.parse;
}
let initializePromise: Promise | undefined;
@@ -52,7 +57,7 @@ const startRunningService = async (wasmURL: string) => {
const wasm = await instantiateWASM(wasmURL, go.importObject);
go.run(wasm.instance);
- const apiKeys = new Set(['transform']);
+ const apiKeys = new Set(['transform', 'parse']);
const service: any = Object.create(null);
for (const key of apiKeys.values()) {
@@ -63,5 +68,6 @@ const startRunningService = async (wasmURL: string) => {
longLivedService = {
transform: (input, options) => new Promise((resolve) => resolve(service.transform(input, options || {}))),
+ parse: (input, options) => new Promise((resolve) => resolve(service.parse(input, options || {}))).then((result: any) => ({ ...result, ast: JSON.parse(result.ast) })),
};
};
diff --git a/lib/compiler/browser/utils.ts b/lib/compiler/browser/utils.ts
new file mode 100644
index 000000000..e62a94178
--- /dev/null
+++ b/lib/compiler/browser/utils.ts
@@ -0,0 +1,66 @@
+import {
+ Node,
+ ParentNode,
+ RootNode,
+ ElementNode,
+ CustomElementNode,
+ ComponentNode,
+ LiteralNode,
+ ExpressionNode,
+ TextNode,
+ CommentNode,
+ DoctypeNode,
+ FrontmatterNode,
+} from '../shared/ast';
+
+export interface Visitor {
+ (node: Node, parent?: ParentNode, index?: number): void | Promise;
+}
+
+function guard(type: string) {
+ return (node: Node): node is Type => node.type === type;
+}
+
+export const is = {
+ parent(node: Node): node is ParentNode {
+ return Array.isArray((node as any).children);
+ },
+ literal(node: Node): node is LiteralNode {
+ return typeof (node as any).value === 'string';
+ },
+ tag(node: Node): node is ElementNode | CustomElementNode | ComponentNode {
+ return node.type === 'element' || node.type === 'custom-element' || node.type === 'component';
+ },
+ whitespace(node: Node): node is TextNode {
+ return node.type === 'text' && node.value.trim().length === 0;
+ },
+ root: guard('root'),
+ element: guard('element'),
+ customElement: guard('custom-element'),
+ component: guard('component'),
+ expression: guard('expression'),
+ text: guard('text'),
+ doctype: guard('doctype'),
+ comment: guard('comment'),
+ frontmatter: guard('frontmatter'),
+};
+
+class Walker {
+ constructor(private callback: Visitor) {}
+ async visit(node: Node, parent?: ParentNode, index?: number): Promise {
+ await this.callback(node, parent, index);
+ if (is.parent(node)) {
+ let promises = [];
+ for (let i = 0; i < node.children.length; i++) {
+ const child = node.children[i];
+ promises.push(this.callback(child, node as ParentNode, i));
+ }
+ await Promise.all(promises);
+ }
+ }
+}
+
+export function walk(node: ParentNode, callback: Visitor): void {
+ const walker = new Walker(callback);
+ walker.visit(node);
+}
diff --git a/lib/compiler/deno/astro.wasm b/lib/compiler/deno/astro.wasm
index a8cceeb54..905bb4c58 100755
Binary files a/lib/compiler/deno/astro.wasm and b/lib/compiler/deno/astro.wasm differ
diff --git a/lib/compiler/node/index.ts b/lib/compiler/node/index.ts
index 013bc8ee1..8f1c83db4 100644
--- a/lib/compiler/node/index.ts
+++ b/lib/compiler/node/index.ts
@@ -7,6 +7,10 @@ export const transform: typeof types.transform = async (input, options) => {
return ensureServiceIsRunning().then((service) => service.transform(input, options));
};
+export const parse: typeof types.parse = async (input, options) => {
+ return ensureServiceIsRunning().then((service) => service.parse(input, options));
+};
+
export const compile = async (template: string): Promise => {
const { default: mod } = await import(`data:text/javascript;charset=utf-8;base64,${Buffer.from(template).toString('base64')}`);
return mod;
@@ -14,6 +18,7 @@ export const compile = async (template: string): Promise => {
interface Service {
transform: typeof types.transform;
+ parse: typeof types.parse;
}
let longLivedService: Service | undefined;
@@ -40,7 +45,7 @@ const startRunningService = async () => {
const wasm = await instantiateWASM(fileURLToPath(new URL('../astro.wasm', import.meta.url)), go.importObject);
go.run(wasm.instance);
- const apiKeys = new Set(['transform']);
+ const apiKeys = new Set(['transform', 'parse']);
const service: any = Object.create(null);
for (const key of apiKeys.values()) {
@@ -51,6 +56,7 @@ const startRunningService = async () => {
longLivedService = {
transform: (input, options) => new Promise((resolve) => resolve(service.transform(input, options || {}))),
+ parse: (input, options) => new Promise((resolve) => resolve(service.parse(input, options || {}))).then((result: any) => ({ ...result, ast: JSON.parse(result.ast) })),
};
return longLivedService;
};
diff --git a/lib/compiler/node/utils.ts b/lib/compiler/node/utils.ts
new file mode 100644
index 000000000..e62a94178
--- /dev/null
+++ b/lib/compiler/node/utils.ts
@@ -0,0 +1,66 @@
+import {
+ Node,
+ ParentNode,
+ RootNode,
+ ElementNode,
+ CustomElementNode,
+ ComponentNode,
+ LiteralNode,
+ ExpressionNode,
+ TextNode,
+ CommentNode,
+ DoctypeNode,
+ FrontmatterNode,
+} from '../shared/ast';
+
+export interface Visitor {
+ (node: Node, parent?: ParentNode, index?: number): void | Promise;
+}
+
+function guard(type: string) {
+ return (node: Node): node is Type => node.type === type;
+}
+
+export const is = {
+ parent(node: Node): node is ParentNode {
+ return Array.isArray((node as any).children);
+ },
+ literal(node: Node): node is LiteralNode {
+ return typeof (node as any).value === 'string';
+ },
+ tag(node: Node): node is ElementNode | CustomElementNode | ComponentNode {
+ return node.type === 'element' || node.type === 'custom-element' || node.type === 'component';
+ },
+ whitespace(node: Node): node is TextNode {
+ return node.type === 'text' && node.value.trim().length === 0;
+ },
+ root: guard('root'),
+ element: guard('element'),
+ customElement: guard('custom-element'),
+ component: guard('component'),
+ expression: guard('expression'),
+ text: guard('text'),
+ doctype: guard('doctype'),
+ comment: guard('comment'),
+ frontmatter: guard('frontmatter'),
+};
+
+class Walker {
+ constructor(private callback: Visitor) {}
+ async visit(node: Node, parent?: ParentNode, index?: number): Promise {
+ await this.callback(node, parent, index);
+ if (is.parent(node)) {
+ let promises = [];
+ for (let i = 0; i < node.children.length; i++) {
+ const child = node.children[i];
+ promises.push(this.callback(child, node as ParentNode, i));
+ }
+ await Promise.all(promises);
+ }
+ }
+}
+
+export function walk(node: ParentNode, callback: Visitor): void {
+ const walker = new Walker(callback);
+ walker.visit(node);
+}
diff --git a/lib/compiler/package.json b/lib/compiler/package.json
index d68d6d777..a92e01f84 100644
--- a/lib/compiler/package.json
+++ b/lib/compiler/package.json
@@ -10,7 +10,7 @@
"build": "tsc -p ."
},
"main": "./node/index.js",
- "types": "./shared/types.d.ts",
+ "types": "./node",
"repository": {
"type": "git",
"url": "https://github.com/withastro/compiler.git"
@@ -21,6 +21,11 @@
"import": "./node/index.js",
"default": "./browser/index.js"
},
+ "./utils": {
+ "browser": "./browser/utils.js",
+ "import": "./node/utils.js",
+ "default": "./browser/utils.js"
+ },
"./astro.wasm": "./astro.wasm"
},
"dependencies": {
diff --git a/lib/compiler/shared/ast.ts b/lib/compiler/shared/ast.ts
new file mode 100644
index 000000000..25f70a498
--- /dev/null
+++ b/lib/compiler/shared/ast.ts
@@ -0,0 +1,85 @@
+export type ParentNode = RootNode | ElementNode | ComponentNode | CustomElementNode | ExpressionNode;
+export type Node = RootNode | ElementNode | ComponentNode | CustomElementNode | ExpressionNode | TextNode | FrontmatterNode | DoctypeNode | CommentNode;
+
+export interface Position {
+ start: Point;
+ end?: Point;
+}
+export interface Point {
+ /** 1-based line number */
+ line: number;
+ /** 1-based column number, per-line */
+ column: number;
+ /** 0-based byte offset */
+ offset: number;
+}
+export interface BaseNode {
+ type: string;
+ position?: Position;
+}
+
+export interface ParentLikeNode extends BaseNode {
+ type: 'element' | 'component' | 'custom-element' | 'expression' | 'root';
+ children: Node[];
+}
+
+export interface LiteralNode extends BaseNode {
+ type: 'text' | 'doctype' | 'comment' | 'frontmatter';
+ value: string;
+}
+
+export interface RootNode extends ParentLikeNode {
+ type: 'root';
+}
+
+export interface AttributeNode extends BaseNode {
+ type: 'attribute';
+ kind: 'quoted' | 'empty' | 'expression' | 'spread' | 'shorthand' | 'template-literal';
+ name: string;
+ value: string;
+}
+
+export interface DirectiveNode extends Omit {
+ type: 'directive';
+}
+
+export interface TextNode extends LiteralNode {
+ type: 'text';
+}
+
+export interface ElementNode extends ParentLikeNode {
+ type: 'element';
+ name: string;
+ attributes: AttributeNode[];
+ directives: DirectiveNode[];
+}
+
+export interface ComponentNode extends ParentLikeNode {
+ type: 'component';
+ name: string;
+ attributes: AttributeNode[];
+ directives: DirectiveNode[];
+}
+
+export interface CustomElementNode extends ParentLikeNode {
+ type: 'custom-element';
+ name: string;
+ attributes: AttributeNode[];
+ directives: DirectiveNode[];
+}
+
+export interface DoctypeNode extends LiteralNode {
+ type: 'doctype';
+}
+
+export interface CommentNode extends LiteralNode {
+ type: 'comment';
+}
+
+export interface FrontmatterNode extends LiteralNode {
+ type: 'frontmatter';
+}
+
+export interface ExpressionNode extends ParentLikeNode {
+ type: 'expression';
+}
diff --git a/lib/compiler/shared/types.ts b/lib/compiler/shared/types.ts
index cbd03300f..e6b722104 100644
--- a/lib/compiler/shared/types.ts
+++ b/lib/compiler/shared/types.ts
@@ -1,8 +1,16 @@
+import { RootNode } from './ast';
+export * from './ast';
+
export interface PreprocessorResult {
code: string;
map?: string;
}
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface ParseOptions {
+ position?: boolean;
+}
+
export interface TransformOptions {
internalURL?: string;
site?: string;
@@ -36,6 +44,10 @@ export interface TransformResult {
map: string;
}
+export interface ParseResult {
+ ast: RootNode;
+}
+
// This function transforms a single JavaScript file. It can be used to minify
// JavaScript, convert TypeScript/JSX to JavaScript, or convert newer JavaScript
// to older JavaScript. It returns a promise that is either resolved with a
@@ -45,6 +57,8 @@ export interface TransformResult {
// Works in browser: yes
export declare function transform(input: string, options?: TransformOptions): Promise;
+export declare function parse(input: string, options?: ParseOptions): Promise;
+
// This configures the browser-based version of astro. It is necessary to
// call this first and wait for the returned promise to be resolved before
// making other API calls when using astro in the browser.
diff --git a/lib/compiler/test/component-name.test.mjs b/lib/compiler/test/component-name.test.mjs
new file mode 100644
index 000000000..025debb31
--- /dev/null
+++ b/lib/compiler/test/component-name.test.mjs
@@ -0,0 +1,16 @@
+/* eslint-disable no-console */
+
+import { transform } from '@astrojs/compiler';
+
+async function run() {
+ const result = await transform(`Hello world!
`, {
+ sourcemap: true,
+ pathname: '/src/components/Cool.astro',
+ });
+
+ if (!result.code.includes('export default $$Cool')) {
+ throw new Error(`Expected component export to be named "Cool"!`);
+ }
+}
+
+await run();
diff --git a/lib/compiler/test/parse.test.mjs b/lib/compiler/test/parse.test.mjs
new file mode 100644
index 000000000..cff4bf02a
--- /dev/null
+++ b/lib/compiler/test/parse.test.mjs
@@ -0,0 +1,42 @@
+/* eslint-disable no-console */
+
+import { parse } from '@astrojs/compiler';
+import { walk, is } from '@astrojs/compiler/utils';
+
+const src = `---
+let value = 'world';
+---
+
+Hello {value}
+`;
+
+async function run() {
+ const result = await parse(src);
+
+ if (typeof result !== 'object') {
+ throw new Error(`Expected "parse" to return an object!`);
+ }
+ if (result.ast.type !== 'root') {
+ throw new Error(`Expected "ast" root node to be of type "root"`);
+ }
+ const [frontmatter, _, element] = result.ast.children;
+ if (frontmatter.type !== 'frontmatter') {
+ throw new Error(`Expected first child node to be of type "frontmatter"`);
+ }
+ if (element.type !== 'element') {
+ throw new Error(`Expected third child node to be of type "element"`);
+ }
+
+ walk(result.ast, (node) => {
+ if (is.tag(node)) {
+ if (node.name !== 'h1') {
+ throw new Error(`Expected element to be ""`);
+ }
+ if (!node.directives || (Array.isArray(node.directives) && !(node.directives.length >= 1))) {
+ throw new Error(`Expected client:load directive to be on ""`);
+ }
+ }
+ });
+}
+
+await run();
diff --git a/lib/compiler/test/test.mjs b/lib/compiler/test/test.mjs
index c9b4b1985..c6f8899ca 100644
--- a/lib/compiler/test/test.mjs
+++ b/lib/compiler/test/test.mjs
@@ -2,7 +2,9 @@ import './basic.test.mjs';
import './body-fragment.test.mjs';
import './body-expression.test.mjs';
import './component-only.test.mjs';
+import './component-name.test.mjs';
import './empty-style.test.mjs';
import './output.test.mjs';
import './script-fragment.test.mjs';
import './top-level-expression.test.mjs';
+import './parse.test.mjs';
diff --git a/lib/compiler/utils.d.ts b/lib/compiler/utils.d.ts
new file mode 100644
index 000000000..644c24206
--- /dev/null
+++ b/lib/compiler/utils.d.ts
@@ -0,0 +1 @@
+export * from './node/utils';