chore: improve python parsing performance

README.md

@@ -29,6 +29,11 @@
     ensuring your CI validates the integrity of your dependency graph, keeping your code base clean.
 </p>
 
+## Checkout the entropy graph of well-known libraries
+- [three.js](https://dep-tree-explorer.vercel.app/api?repo=https%3A%2F%2Fgithub.com%2Fmrdoob%2Fthree.js&entrypoint=src%2FThree.js)
+- [langchain](https://dep-tree-explorer.vercel.app/api?repo=https%3A%2F%2Fgithub.com%2Flangchain-ai%2Flangchain&entrypoint=libs%2Flangchain%2Flangchain%2F__init__.py)
+- [vuejs](https://dep-tree-explorer.vercel.app/api?repo=https%3A%2F%2Fgithub.com%2Fvuejs%2Fvue&entrypoint=src%2Fcore%2Findex.ts)
+
 ## Install
 
 On Mac and Linux, it can be installed using brew:

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/gammazero/deque v0.2.1
 	github.com/gdamore/tcell/v2 v2.7.0
 	github.com/go-git/go-git/v5 v5.11.0
+	github.com/schollz/progressbar/v3 v3.14.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
@@ -34,6 +35,7 @@ require (
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect

go.sum

@@ -58,6 +58,7 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -69,8 +70,11 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
@@ -86,6 +90,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI=
+github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -97,6 +103,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
@@ -145,13 +152,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/dep_tree/dep_tree.go

@@ -2,8 +2,10 @@ package dep_tree
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/elliotchance/orderedmap/v2"
+	"github.com/schollz/progressbar/v3"
 
 	"github.com/gabotechs/dep-tree/internal/graph"
 )
@@ -30,15 +32,36 @@ type DepTree[T any] struct {
 	Cycles *orderedmap.OrderedMap[[2]string, graph.Cycle]
 	// just some internal cache.
 	root *graph.Node[T]
+	// callbacks
+	onStartLoading func()
+	onNodeLoad     func(*graph.Node[T])
+	onFinishLoad   func()
 }
 
 func NewDepTree[T any](parser NodeParser[T]) *DepTree[T] {
-	return &DepTree[T]{
+	return (&DepTree[T]{
 		NodeParser: parser,
 		Nodes:      []*DepTreeNode[T]{},
 		Graph:      graph.NewGraph[T](),
 		Cycles:     orderedmap.NewOrderedMap[[2]string, graph.Cycle](),
+	}).withLoader()
+}
+
+func (dt *DepTree[T]) withLoader() *DepTree[T] {
+	bar := progressbar.Default(-1)
+	dt.onStartLoading = func() {
+		bar.Reset()
+	}
+	dt.onNodeLoad = func(n *graph.Node[T]) {
+		_ = bar.Add(1)
+		bar.Describe(fmt.Sprintf("Loading %s...", dt.NodeParser.Display(n)))
+	}
+	dt.onFinishLoad = func() {
+		bar.Describe("Finished loading")
+		_ = bar.Finish()
+		_ = bar.Clear()
 	}
+	return dt
 }
 
 func (dt *DepTree[T]) Root() (*graph.Node[T], error) {

internal/dep_tree/deps.go

@@ -5,8 +5,9 @@ import (
 	"sort"
 
 	"github.com/elliotchance/orderedmap/v2"
-	"github.com/gabotechs/dep-tree/internal/graph"
 	"github.com/gammazero/deque"
+
+	"github.com/gabotechs/dep-tree/internal/graph"
 )
 
 func (dt *DepTree[T]) LoadDeps(ctx context.Context) (context.Context, error) {
@@ -33,12 +34,14 @@ func (dt *DepTree[T]) LoadGraph(ctx context.Context) (context.Context, error) {
 	dt.Graph.AddNode(root)
 
 	visited := make(map[string]bool)
+	dt.onStartLoading()
 
 	for queue.Len() > 0 {
 		node := queue.PopFront()
 		if _, ok := visited[node.Id]; ok {
 			continue
 		}
+		dt.onNodeLoad(node)
 		visited[node.Id] = true
 
 		newCtx, deps, err := dt.NodeParser.Deps(ctx, node)
@@ -63,6 +66,7 @@ func (dt *DepTree[T]) LoadGraph(ctx context.Context) (context.Context, error) {
 			}
 		}
 	}
+	dt.onFinishLoad()
 
 	return ctx, nil
 }

internal/python/exports.go

@@ -58,7 +58,7 @@ func (l *Language) ParseExports(file *python_grammar.File) (*language.ExportsEnt
 			}
 			exports = append(exports, entry)
 
-		case stmt.VariableUnpack != nil && !stmt.VariableUnpack.Indented:
+		case stmt.VariableUnpack != nil:
 			entry := language.ExportEntry{
 				Names: make([]language.ExportName, len(stmt.VariableUnpack.Names)),
 				Path:  file.Path,
@@ -67,7 +67,7 @@ func (l *Language) ParseExports(file *python_grammar.File) (*language.ExportsEnt
 				entry.Names[i] = language.ExportName{Original: name}
 			}
 			exports = append(exports, entry)
-		case stmt.VariableAssign != nil && !stmt.VariableAssign.Indented:
+		case stmt.VariableAssign != nil:
 			entry := language.ExportEntry{
 				Names: make([]language.ExportName, len(stmt.VariableAssign.Names)),
 				Path:  file.Path,
@@ -76,12 +76,12 @@ func (l *Language) ParseExports(file *python_grammar.File) (*language.ExportsEnt
 				entry.Names[i] = language.ExportName{Original: name}
 			}
 			exports = append(exports, entry)
-		case stmt.VariableTyping != nil && !stmt.VariableTyping.Indented:
+		case stmt.VariableTyping != nil:
 			exports = append(exports, language.ExportEntry{
 				Names: []language.ExportName{{Original: stmt.VariableTyping.Name}},
 				Path:  file.Path,
 			})
-		case stmt.Function != nil && !stmt.Function.Indented:
+		case stmt.Function != nil:
 			exports = append(exports, language.ExportEntry{
 				Names: []language.ExportName{
 					{
@@ -90,7 +90,7 @@ func (l *Language) ParseExports(file *python_grammar.File) (*language.ExportsEnt
 				},
 				Path: file.Path,
 			})
-		case stmt.Class != nil && !stmt.Class.Indented:
+		case stmt.Class != nil:
 			exports = append(exports, language.ExportEntry{
 				Names: []language.ExportName{
 					{

internal/python/python_grammar/export.go

@@ -2,26 +2,21 @@
 package python_grammar
 
 type VariableUnpack struct {
-	Indented bool     `@Space?`
-	Names    []string `( (@Ident (Space? "," Space? @Ident)+) | ( "(" (Space|NewLine)* @Ident ( (Space|NewLine)* "," (Space|NewLine)* @Ident )+ (Space|NewLine)* ")" ) ) Space? "="`
+	Names []string `(NewLine | SOF) ( (@Ident (Space? "," Space? @Ident)+) | ( "(" (Space|NewLine)* @Ident ( (Space|NewLine)* "," (Space|NewLine)* @Ident )+ (Space|NewLine)* ")" ) ) Space? "="`
 }
 
 type VariableTyping struct {
-	Indented bool   `@Space?`
-	Name     string `@Ident Space? ":" Space? Ident`
+	Name string `(NewLine | SOF) @Ident Space? ":" Space? Ident`
 }
 
 type VariableAssign struct {
-	Indented bool     `@Space?`
-	Names    []string `(@Ident Space? "=" Space?)+`
+	Names []string `(NewLine | SOF) (@Ident Space? "=" Space?)+`
 }
 
 type Function struct {
-	Indented bool   `@Space?`
-	Name     string `("async" Space)? "def" Space @Ident`
+	Name string `(NewLine | SOF) ("async" Space)? "def" Space @Ident`
 }
 
 type Class struct {
-	Indented bool   `@Space?`
-	Name     string `"class" Space @Ident`
+	Name string `(NewLine | SOF) "class" Space @Ident`
 }

internal/python/python_grammar/export_test.go

@@ -30,8 +30,7 @@ func TestExport(t *testing.T) {
 			Name: "foo",
 		},
 		{
-			Name:                    " foo = 1",
-			ExpectedVariableAssigns: []VariableAssign{{Names: []string{"foo"}, Indented: true}},
+			Name: " foo = 1",
 		},
 		{
 			Name:                    "foo: int = 1",
@@ -46,8 +45,7 @@ func TestExport(t *testing.T) {
 			ExpectedVariableAssigns: []VariableAssign{{Names: []string{"foo", "bar"}}},
 		},
 		{
-			Name:                    " foo = bar = 1",
-			ExpectedVariableAssigns: []VariableAssign{{Names: []string{"foo", "bar"}, Indented: true}},
+			Name: " foo = bar = 1",
 		},
 		{
 			Name:                    "foo, bar = 1, 1",
@@ -62,8 +60,7 @@ func TestExport(t *testing.T) {
 			ExpectedVariableUnpacks: []VariableUnpack{{Names: []string{"foo", "bar"}}},
 		},
 		{
-			Name:                    " (\n  foo,\n  bar\n) = 1, 1",
-			ExpectedVariableUnpacks: []VariableUnpack{{Names: []string{"foo", "bar"}, Indented: true}},
+			Name: " (\n  foo,\n  bar\n) = 1, 1",
 		},
 		{
 			Name:              "def func():",
@@ -74,8 +71,7 @@ func TestExport(t *testing.T) {
 			ExpectedFunctions: []Function{{Name: "func"}},
 		},
 		{
-			Name:              " def func():",
-			ExpectedFunctions: []Function{{Name: "func", Indented: true}},
+			Name: " def func():",
 		},
 		{
 			Name:            "class cls:",
@@ -86,8 +82,7 @@ func TestExport(t *testing.T) {
 			ExpectedClasses: []Class{{Name: "cls"}},
 		},
 		{
-			Name:            " class cls:",
-			ExpectedClasses: []Class{{Name: "cls", Indented: true}},
+			Name: " class cls:",
 		},
 	}
 

internal/python/python_grammar/grammar.go

@@ -22,7 +22,7 @@ type Statement struct {
 }
 
 type File struct {
-	Statements []*Statement `(@@ | ANY | ALL | Ident | Space | NewLine | String | MultilineString)*`
+	Statements []*Statement `(@@ | SOF | ANY | ALL | Ident | Space | NewLine | String | MultilineString)*`
 	Path       string
 	loc        int
 	size       int
@@ -37,23 +37,23 @@ func (f File) Size() int {
 }
 
 var (
-	lex = lexer.MustSimple(
+	lex = LexerWithSOF(lexer.MustSimple(
 		[]lexer.SimpleRule{
 			{"ALL", `\*`},
 			{"Ident", `[_$a-zA-Z][_$a-zA-Z0-9]*`},
 			{"MultilineString", `'''(.|\n)*?'''` + "|" + `"""(.|\n)*?"""`},
 			{"String", `'(?:\\.|[^'])*'` + "|" + `"(?:\\.|[^"])*"`},
 			// https://stackoverflow.com/questions/69184441/regular-expression-for-comments-in-python-re
 			{"Comment", `#.*`},
-			{"NewLine", `\n`},
+			{"NewLine", `[\n\r]`},
 			{"Space", `\s+`},
 			{"ANY", `.`},
 		},
-	)
+	))
 	parser = participle.MustBuild[File](
 		participle.Lexer(lex),
 		participle.Elide("Comment", "String", "MultilineString"),
-		participle.UseLookahead(1024),
+		participle.UseLookahead(64),
 	)
 )
 

internal/python/python_grammar/import.go

@@ -11,7 +11,7 @@ type FromImport struct {
 	Relative []bool         `"from" (Space | @".") @"."*`
 	Path     []string       `@Ident? (Space? "." Space? @Ident)* Space`
 	All      bool           `"import" Space ( @ALL |`
-	Names    []ImportedName `( (@@ (Space? "," Space? @@)*) | ( "(" (Space|NewLine)* @@ ( (Space|NewLine)* "," (Space|NewLine)* @@ )* (Space|NewLine)* ")" ) ) )`
+	Names    []ImportedName `( (@@ (Space? "," Space? @@)*) | ( "(" (Space|NewLine)* @@ ( (Space|NewLine)* "," (Space|NewLine)* @@ )* (Space|NewLine)* ","? (Space|NewLine)* ")" ) ) )`
 }
 
 type Import struct {

internal/python/python_grammar/import_test.go

@@ -75,6 +75,10 @@ func TestImport(t *testing.T) {
 			Name:                "from foo import (\n    a,\n    b\n)",
 			ExpectedFromImports: []FromImport{{Names: []ImportedName{{Name: "a"}, {Name: "b"}}, Path: []string{"foo"}}},
 		},
+		{
+			Name:                "from foo import (\n    a,\n    b,\n)",
+			ExpectedFromImports: []FromImport{{Names: []ImportedName{{Name: "a"}, {Name: "b"}}, Path: []string{"foo"}}},
+		},
 		{
 			Name:                "from foo.bar import (a)",
 			ExpectedFromImports: []FromImport{{Names: []ImportedName{{Name: "a"}}, Path: []string{"foo", "bar"}}},