From 28d92fc6d7191a4a4b46c793d91a213a2262a8c4 Mon Sep 17 00:00:00 2001 From: Gabriel Musat Date: Sun, 18 Feb 2024 10:38:42 +0100 Subject: [PATCH] Add a guide for implementing new languages --- README.md | 4 + cmd/root.go | 10 ++ docs/IMPLEMENTING_NEW_LANGUAGES.md | 251 +++++++++++++++++++++++++++++ internal/dummy/language.go | 63 ++++++++ internal/dummy/parser.go | 39 +++++ internal/dummy/parser_test.go | 58 +++++++ internal/js/exports.go | 32 ++-- internal/js/exports_test.go | 12 +- internal/language/exports.go | 57 +++---- internal/language/exports_test.go | 22 +-- internal/language/language.go | 56 +++++-- internal/language/parser.go | 4 +- internal/language/parser_test.go | 84 +++++----- internal/language/test_language.go | 6 +- internal/python/exports.go | 40 ++--- internal/python/exports_test.go | 60 +++---- internal/rust/exports.go | 20 +-- internal/rust/exports_test.go | 28 ++-- 18 files changed, 645 insertions(+), 201 deletions(-) create mode 100644 docs/IMPLEMENTING_NEW_LANGUAGES.md create mode 100644 internal/dummy/language.go create mode 100644 internal/dummy/parser.go create mode 100644 internal/dummy/parser_test.go diff --git a/README.md b/README.md index 6aa59df..9076cb9 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,10 @@ npm install @dep-tree/cli +If you want to contribute additional languages, there's a guide [here](docs/IMPLEMENTING_NEW_LANGUAGES.md) +that teaches how to implement new languages with a hands-on example based on a fictional language. +Contributions are always welcome! + ## About Dep Tree `dep-tree` is a cli tool for visualizing the complexity of a code base, and creating diff --git a/cmd/root.go b/cmd/root.go index a406397..5ced5bb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/gabotechs/dep-tree/internal/config" + "github.com/gabotechs/dep-tree/internal/dummy" "github.com/gabotechs/dep-tree/internal/js" "github.com/gabotechs/dep-tree/internal/language" "github.com/gabotechs/dep-tree/internal/python" @@ -97,6 +98,7 @@ func inferLang(files []string, cfg *config.Config) (language.Language, error) { js int python int rust int + dummy int }{} top := struct { lang string @@ -122,6 +124,12 @@ func inferLang(files []string, cfg *config.Config) (language.Language, error) { top.v = score.python top.lang = "python" } + case utils.EndsWith(file, dummy.Extensions): + score.dummy += 1 + if score.dummy > top.v { + top.v = score.dummy + top.lang = "dummy" + } } } if top.lang == "" { @@ -134,6 +142,8 @@ func inferLang(files []string, cfg *config.Config) (language.Language, error) { return rust.MakeRustLanguage(&cfg.Rust) case "python": return python.MakePythonLanguage(&cfg.Python) + case "dummy": + return &dummy.Language{}, nil default: return nil, fmt.Errorf("file \"%s\" not supported", files[0]) } diff --git a/docs/IMPLEMENTING_NEW_LANGUAGES.md b/docs/IMPLEMENTING_NEW_LANGUAGES.md new file mode 100644 index 0000000..aea3688 --- /dev/null +++ b/docs/IMPLEMENTING_NEW_LANGUAGES.md @@ -0,0 +1,251 @@ +# Implementing new languages in Dep Tree + +Implementing a new language in Dep Tree boils down to writing code that satisfies the +[`Language` interface](../internal/language/language.go) and wiring it up to the appropriate +file extensions. There's three core methods that should be provided: + +- ParseFile: parses a file object given its path. +- ParseImports: given a parsed file, retrieves the imported symbols, like functions, classes or variables. +- ParseExports: given a parsed file, retrieves the exported symbols. + +As long as implementations are able to satisfy this interface, they will be compatible with dep-tree's +machinery for creating graphs and analyzing dependencies. + +## Learn by example + +First, clone Dep Tree's repository, as we will work directly committing files to it: + +```shell +git clone https://github.com/gabotechs/dep-tree +``` + +Then, ensure you have Golang set-up in your machine. Dep Tree is written in Golang, so this +tutorial will assume that you have the compiler installed and that you have some basic knowledge +of the language. + +### The Dummy Language + +In order to keep it simple, we will create a fictional programming language +that only has `import` and `export` statements, we will call it "Dummy Language", and its file +extension will be `.dl`. + +Dummy Language files will have statements like this: +```js +import foo from ./file.dl + +export bar +``` + +In this file, the first statement imports symbol `foo` from the file `file.dl` located in the +same folder, and the second statement is exporting the symbol `bar`. We can expect `file.dl` +to contain something like this: +```js +export foo +``` +Where foo is the symbol that the other file is trying to import. + +### 1. Parsing files + +First, we will need to create a parser for our Dummy Language. There are many tools in Golang for +creating language parsers, but most language implementations in Dep Tree use https://github.com/alecthomas/participle, +which allows writing parsers with very few lines of code. + +Navigate to Dep Tree's cloned repository, and create a directory under the `internal` folder called `dummy`. +Create a file called `parser.go` inside `internal/dummy`, where we will place our parser code: + +```go +package dummy + +import ( + "github.com/alecthomas/participle/v2" + "github.com/alecthomas/participle/v2/lexer" +) + +type ImportStatement struct { + Symbols []string `"import" @Ident ("," @Ident)*` + From string `"from" @(Ident|Punctuation|"/")*` +} + +type ExportStatement struct { + Symbol string `"export" @Ident` +} + +type Statement struct { + Import *ImportStatement `@@ |` + Export *ExportStatement `@@` +} + +type File struct { + Statements []Statement `@@*` +} + +var ( + lex = lexer.MustSimple( + []lexer.SimpleRule{ + {"KewWord", "(export|import|from)"}, + {"Punctuation", `[,\./]`}, + {"Ident", `[a-zA-Z]+`}, + {"Whitespace", `\s+`}, + }, + ) + parser = participle.MustBuild[File]( + participle.Lexer(lex), + participle.Elide("Whitespace"), + ) +) +``` + +We will not cover here how [participle](https://github.com/alecthomas/participle) works, but +it's important to note that using it is not required. If you are implementing a new language +for Dep Tree, feel free to choose the parsing mechanism that you find most suitable. + +We now need to implement the `ParseFile` method from the `Language` interface. + +We will place all our methods in a file called `language.go` inside the `internal/dummy` dir: + +```go +package dummy + +import ( + "bytes" + "os" + "path/filepath" + + "github.com/gabotechs/dep-tree/internal/language" +) + +type Language struct{} + +func (l *Language) ParseFile(path string) (*language.FileInfo, error) { + //TODO implement me + panic("implement me") +} +``` + +The ultimate goal of the `ParseFile` method is to output a `FileInfo` struct, that contains +information about the source file itself, like its size, the amount of lines of code it has, its +parsed statements, it's path on the disk... + +A fully working implementation of this method would look like this: +```go +func (l *Language) ParseFile(path string) (*language.FileInfo, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + file, err := parser.ParseBytes(path, content) + if err != nil { + return nil, err + } + currentDir, _ := os.Getwd() + relPath, _ := filepath.Rel(currentDir, path) + return &language.FileInfo{ + Content: file.Statements, // dump the parsed statements into the FileInfo struct. + Loc: bytes.Count(content, []byte("\n")), // get the amount of lines of code. + Size: len(content), // get the size of the file in bytes. + AbsPath: path, // provide its absolute path. + RelPath: relPath, // provide the path relative to the current dir. + }, nil +} +``` +The `RelPath` attribute is important as it's what ultimately will be shown while rendering the graph. +Some language implementations choose to provide a path not relative to the current working directory, +but to its closest `package.json` for example. Language implementation are free to choose what `RelPath` +should look like. + +### 2. Parsing Import statements + +Parsing imports is far simpler, as we have everything in place already. + +This method accepts the same `FileInfo` structure that we created previously in the `ParseFile` method, +and returns an `ImportResult` structure with all the import statements gathered from the file. + +We will place our method implementation in the same `language.go` file, just below the `ParseFile` method: + +```go +func (l *Language) ParseImports(file *language.FileInfo) (*language.ImportsResult, error) { + var result language.ImportsResult + + for _, statement := range file.Content.([]Statement) { + if statement.Import != nil { + result.Imports = append(result.Imports, language.ImportEntry{ + Symbols: statement.Import.Symbols, + // in our Dummy language, imports are always relative to source file. + AbsPath: filepath.Join(filepath.Dir(file.AbsPath), statement.Import.From), + }) + } + } + + return &result, nil +} +``` + +### 3. Parsing Export statements + +The `ParseExports` method is very similar to the `ParseImports` method, but it gathers export statements rather +than import statements. + +```go +func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsResult, error) { + var result language.ExportsResult + + for _, statement := range file.Content.([]Statement) { + if statement.Export != nil { + result.Exports = append(result.Exports, language.ExportEntry{ + // our Dummy Language only allows exporting 1 symbol at a time, and does not support aliasing. + Symbols: []language.ExportSymbol{{Original: statement.Export.Symbol}}, + AbsPath: file.AbsPath, + }) + } + } + + return &result, nil +} +``` + +### 4. Wiring up the language with Dep Tree + +Now that the `Language` interface is fully implemented, we need to wire it up so that it's recognized by +Dep Tree. For that, let's declare the array of extensions that the Dummy Language supports in the +`internal/dummy/language.go` file: + +```go +var Extensions = []string{"dl"} +``` + +Now, we will need to go to `cmd/root.go` and tweak the `inferLang` function in order to also take `.dl` files +into account. Beware that this function is highly susceptible to changing, so the following instructions +might not be accurate: + +- Add one more entry to the `score` struct: +```go + score := struct { + js int + python int + rust int + + dummy int // <- add this + }{} +``` +- Add one case branch in the `for` loop: +```go + + case utils.EndsWith(file, dummy.Extensions): + + score.dummy += 1 + + if score.dummy > top.v { + + top.v = score.dummy + + top.lang = "dummy" + + } +``` +- Add one case branch at the bottom of the function +```go + + case "dummy": + + return &dummy.Language{}, nil +``` + +### 5. Running Dep Tree on the Dummy Language + +You have everything in place to start playing with the Dummy Language and Dep Tree. +- Compile Dep Tree by running `go build` in the root directory of the project +- Create some Dummy Language files that import each other +- use the generated binary `./dep-tree` and run them on one of the Dummy Language files + +If everything went correctly, you should be seeing a graph that renders your files. diff --git a/internal/dummy/language.go b/internal/dummy/language.go new file mode 100644 index 0000000..f61233c --- /dev/null +++ b/internal/dummy/language.go @@ -0,0 +1,63 @@ +package dummy + +import ( + "bytes" + "os" + "path/filepath" + + "github.com/gabotechs/dep-tree/internal/language" +) + +type Language struct{} + +func (l *Language) ParseFile(path string) (*language.FileInfo, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + file, err := parser.ParseBytes(path, content) + if err != nil { + return nil, err + } + currentDir, _ := os.Getwd() + relPath, _ := filepath.Rel(currentDir, path) + return &language.FileInfo{ + Content: file.Statements, + Loc: bytes.Count(content, []byte("\n")), + Size: len(content), + AbsPath: path, + RelPath: relPath, + }, nil +} + +func (l *Language) ParseImports(file *language.FileInfo) (*language.ImportsResult, error) { + var result language.ImportsResult + + for _, statement := range file.Content.([]Statement) { + if statement.Import != nil { + result.Imports = append(result.Imports, language.ImportEntry{ + Symbols: statement.Import.Symbols, + AbsPath: filepath.Join(filepath.Dir(file.AbsPath), statement.Import.From), + }) + } + } + + return &result, nil +} + +func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsResult, error) { + var result language.ExportsResult + + for _, statement := range file.Content.([]Statement) { + if statement.Export != nil { + result.Exports = append(result.Exports, language.ExportEntry{ + Symbols: []language.ExportSymbol{{Original: statement.Export.Symbol}}, + AbsPath: file.AbsPath, + }) + } + } + + return &result, nil +} + +var Extensions = []string{"dl"} diff --git a/internal/dummy/parser.go b/internal/dummy/parser.go new file mode 100644 index 0000000..95177de --- /dev/null +++ b/internal/dummy/parser.go @@ -0,0 +1,39 @@ +package dummy + +import ( + "github.com/alecthomas/participle/v2" + "github.com/alecthomas/participle/v2/lexer" +) + +type ImportStatement struct { + Symbols []string `"import" @Ident ("," @Ident)*` + From string `"from" @(Ident|Punctuation|"/")*` +} + +type ExportStatement struct { + Symbol string `"export" @Ident` +} + +type Statement struct { + Import *ImportStatement `@@ |` + Export *ExportStatement `@@` +} + +type File struct { + Statements []Statement `@@*` +} + +var ( + lex = lexer.MustSimple( + []lexer.SimpleRule{ + {"KewWord", "(export|import|from)"}, + {"Punctuation", `[,\./]`}, + {"Ident", `[a-zA-Z]+`}, + {"Whitespace", `\s+`}, + }, + ) + parser = participle.MustBuild[File]( + participle.Lexer(lex), + participle.Elide("Whitespace"), + ) +) diff --git a/internal/dummy/parser_test.go b/internal/dummy/parser_test.go new file mode 100644 index 0000000..051a8fc --- /dev/null +++ b/internal/dummy/parser_test.go @@ -0,0 +1,58 @@ +package dummy + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParser(t *testing.T) { + tests := []struct { + Name string + Statements []Statement + }{ + { + Name: "import foo from bar", + Statements: []Statement{{ + Import: &ImportStatement{[]string{"foo"}, "bar"}, + }}, + }, + { + Name: "import foo, baz from ./bar.dl", + Statements: []Statement{{ + Import: &ImportStatement{[]string{"foo", "baz"}, "./bar.dl"}, + }}, + }, + { + Name: "export foo", + Statements: []Statement{{ + Export: &ExportStatement{"foo"}, + }}, + }, + + { + Name: "import foo, baz from ./bar.dl", + Statements: []Statement{{ + Import: &ImportStatement{[]string{"foo", "baz"}, "./bar.dl"}, + }}, + }, + { + Name: "import foo, baz from ./bar.dl\n\nexport foo", + Statements: []Statement{ + {Import: &ImportStatement{[]string{"foo", "baz"}, "./bar.dl"}}, + {Export: &ExportStatement{"foo"}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + a := require.New(t) + + result, err := parser.ParseBytes("", []byte(tt.Name)) + a.NoError(err) + + a.Equal(tt.Statements, result.Statements) + }) + } +} diff --git a/internal/js/exports.go b/internal/js/exports.go index 91e1ea2..e1aa6eb 100644 --- a/internal/js/exports.go +++ b/internal/js/exports.go @@ -9,7 +9,7 @@ import ( type ExportsCacheKey string -func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntries, error) { +func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsResult, error) { exports := make([]language.ExportEntry, 0) var errors []error @@ -20,36 +20,36 @@ func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntri // Is this even possible? case stmt.DeclarationExport != nil: exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{ + Symbols: []language.ExportSymbol{ { Original: stmt.DeclarationExport.Name, }, }, - Path: file.AbsPath, + AbsPath: file.AbsPath, }) case stmt.ListExport != nil: if stmt.ListExport.ExportDeconstruction != nil { for _, name := range stmt.ListExport.ExportDeconstruction.Names { exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{ + Symbols: []language.ExportSymbol{ { Original: name.Original, Alias: name.Alias, }, }, - Path: file.AbsPath, + AbsPath: file.AbsPath, }) } } case stmt.DefaultExport != nil: if stmt.DefaultExport.Default { exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{ + Symbols: []language.ExportSymbol{ { Original: "default", }, }, - Path: file.AbsPath, + AbsPath: file.AbsPath, }) } case stmt.ProxyExport != nil: @@ -65,36 +65,36 @@ func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntri case stmt.ProxyExport.ExportAll: if stmt.ProxyExport.ExportAllAlias != "" { exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{ + Symbols: []language.ExportSymbol{ { Original: stmt.ProxyExport.ExportAllAlias, }, }, - Path: exportFrom, + AbsPath: exportFrom, }) } else { exports = append(exports, language.ExportEntry{ - All: true, - Path: exportFrom, + All: true, + AbsPath: exportFrom, }) } case stmt.ProxyExport.ExportDeconstruction != nil: - names := make([]language.ExportName, 0) + names := make([]language.ExportSymbol, 0) for _, name := range stmt.ProxyExport.ExportDeconstruction.Names { - names = append(names, language.ExportName{ + names = append(names, language.ExportSymbol{ Original: name.Original, Alias: name.Alias, }) } exports = append(exports, language.ExportEntry{ - Names: names, - Path: exportFrom, + Symbols: names, + AbsPath: exportFrom, }) } } } - return &language.ExportsEntries{ + return &language.ExportsResult{ Exports: exports, Errors: errors, }, nil diff --git a/internal/js/exports_test.go b/internal/js/exports_test.go index 6fe8958..ecdd9a7 100644 --- a/internal/js/exports_test.go +++ b/internal/js/exports_test.go @@ -26,16 +26,16 @@ func TestParser_parseExports(t *testing.T) { File: filepath.Join(exportsTestFolder, "src", "index.js"), Expected: []language.ExportEntry{ { - All: true, - Path: filepath.Join(cwd, exportsTestFolder, "src", "utils", "index.js"), + All: true, + AbsPath: filepath.Join(cwd, exportsTestFolder, "src", "utils", "index.js"), }, { - Names: []language.ExportName{{Original: "Unexisting"}, {Original: "UnSorter", Alias: "UnSorterAlias"}}, - Path: filepath.Join(cwd, exportsTestFolder, "src", "utils", "index.js"), + Symbols: []language.ExportSymbol{{Original: "Unexisting"}, {Original: "UnSorter", Alias: "UnSorterAlias"}}, + AbsPath: filepath.Join(cwd, exportsTestFolder, "src", "utils", "index.js"), }, { - Names: []language.ExportName{{Original: "aliased"}}, - Path: filepath.Join(cwd, exportsTestFolder, "src", "utils", "unsort.js"), + Symbols: []language.ExportSymbol{{Original: "aliased"}}, + AbsPath: filepath.Join(cwd, exportsTestFolder, "src", "utils", "unsort.js"), }, }, ExpectedErrors: []string{ diff --git a/internal/language/exports.go b/internal/language/exports.go index 013e187..285b271 100644 --- a/internal/language/exports.go +++ b/internal/language/exports.go @@ -9,12 +9,7 @@ import ( "github.com/gabotechs/dep-tree/internal/utils" ) -type ExportName struct { - Original string - Alias string -} - -func (en *ExportName) name() string { +func (en *ExportSymbol) name() string { if en.Alias != "" { return en.Alias } else { @@ -22,20 +17,20 @@ func (en *ExportName) name() string { } } -type ExportEntry struct { - // All: all the names from Path are exported. - All bool - // Names: exported specific names from Path. - Names []ExportName - // Path: absolute path from where they are exported, it might be from the same file or from another. - Path string -} - -type ExportsEntries struct { - // Exports: array of ExportEntry - // NOTE: even though it could work returning a path relative to the file, it should return absolute. - Exports []ExportEntry - // Errors: errors while parsing exports. +// ExportEntries is the result of gathering all the export statements from +// a source file, in case the language implementation explicitly exports certain files. +type ExportEntries struct { + // Symbols is an ordered map data structure where the keys are the symbols exported from + // the source file and the values are path from where they are declared. Symbols might + // be declared in a different path from where they are exported, for example: + // + // export { foo } from './bar' + // + // the `foo` symbol is being exported from the current file, but it's declared on the + // `bar.ts` file. + Symbols *orderedmap.OrderedMap[string, string] + // Errors are the non-fatal errors that occurred while parsing exports. These + // might be rendered nicely in a UI. Errors []error } @@ -43,7 +38,7 @@ func (p *Parser) parseExports( id string, unwrappedExports bool, stack *utils.CallStack, -) (*ExportsResult, error) { +) (*ExportEntries, error) { if stack == nil { stack = utils.NewCallStack() } @@ -70,15 +65,15 @@ func (p *Parser) parseExports( var exportErrors []error for _, export := range wrapped.Exports { - if export.Path == id { - for _, name := range export.Names { - exports.Set(name.name(), export.Path) + if export.AbsPath == id { + for _, name := range export.Symbols { + exports.Set(name.name(), export.AbsPath) } continue } - var unwrapped *ExportsResult - unwrapped, err = p.parseExports(export.Path, unwrappedExports, stack) + var unwrapped *ExportEntries + unwrapped, err = p.parseExports(export.AbsPath, unwrappedExports, stack) if err != nil { exportErrors = append(exportErrors, err) continue @@ -89,28 +84,28 @@ func (p *Parser) parseExports( if unwrappedExports { exports.Set(el.Key, el.Value) } else { - exports.Set(el.Key, export.Path) + exports.Set(el.Key, export.AbsPath) } } continue } exportErrors = append(exportErrors, unwrapped.Errors...) - for _, name := range export.Names { + for _, name := range export.Symbols { if exportPath, ok := unwrapped.Symbols.Get(name.Original); ok { if unwrappedExports { exports.Set(name.name(), exportPath) } else { - exports.Set(name.name(), export.Path) + exports.Set(name.name(), export.AbsPath) } } else { - exports.Set(name.name(), export.Path) + exports.Set(name.name(), export.AbsPath) // errors = append(errors, fmt.Errorf(`name "%s" exported in "%s" from "%s" cannot be found in origin file`, name.Original, id, export.Id)). } } } - result := ExportsResult{Symbols: exports, Errors: exportErrors} + result := ExportEntries{Symbols: exports, Errors: exportErrors} p.ExportsCache[cacheKey] = &result return &result, nil } diff --git a/internal/language/exports_test.go b/internal/language/exports_test.go index d1b3365..f7118ab 100644 --- a/internal/language/exports_test.go +++ b/internal/language/exports_test.go @@ -10,36 +10,36 @@ import ( "github.com/stretchr/testify/require" ) -type ExportsResultBuilder map[string]*ExportsEntries +type ExportsResultBuilder map[string]*ExportsResult -func (e *ExportsResultBuilder) Build() map[string]*ExportsEntries { +func (e *ExportsResultBuilder) Build() map[string]*ExportsResult { return *e } func (e *ExportsResultBuilder) Entry(inId string, toId string, names ...string) *ExportsResultBuilder { - var result *ExportsEntries + var result *ExportsResult var ok bool if result, ok = (*e)[inId]; !ok { - result = &ExportsEntries{} + result = &ExportsResult{} } if len(names) == 1 && names[0] == "*" { result.Exports = append(result.Exports, ExportEntry{ - All: true, - Path: toId, + All: true, + AbsPath: toId, }) } else { - var n []ExportName + var n []ExportSymbol for _, name := range names { if strings.HasPrefix(name, "as ") { n[len(n)-1].Alias = strings.TrimLeft(name, "as ") } else { - n = append(n, ExportName{Original: name}) + n = append(n, ExportSymbol{Original: name}) } } result.Exports = append(result.Exports, ExportEntry{ - Names: n, - Path: toId, + Symbols: n, + AbsPath: toId, }) } (*e)[inId] = result @@ -86,7 +86,7 @@ func TestParser_CachedUnwrappedParseExports(t *testing.T) { tests := []struct { Name string Path string - Exports map[string]*ExportsEntries + Exports map[string]*ExportsResult ExpectedUnwrapped *orderedmap.OrderedMap[string, string] ExpectedWrapped *orderedmap.OrderedMap[string, string] ExpectedErrors []string diff --git a/internal/language/language.go b/internal/language/language.go index 63ddcb1..f1115a1 100644 --- a/internal/language/language.go +++ b/internal/language/language.go @@ -1,7 +1,5 @@ package language -import "github.com/elliotchance/orderedmap/v2" - // FileInfo gathers all the information related to a source file. type FileInfo struct { // Content is a bucket for language implementations to inject some language-specific data in it, @@ -24,7 +22,7 @@ type FileInfo struct { Size int } -// ImportEntry represent an import statement in a programming language. +// ImportEntry represents an import statement in a programming language. type ImportEntry struct { // All is true if all the symbols from another source file are imported. Some programming languages // allow importing all the symbols: @@ -60,8 +58,7 @@ func SymbolsImport(symbols []string, absPath string) ImportEntry { return ImportEntry{Symbols: symbols, AbsPath: absPath} } -// ImportsResult is the result of gathering all the import statements from -// a source file. +// ImportsResult is the result of gathering all the import statements from a source file. type ImportsResult struct { // Imports is the list of ImportEntry for the source file. Imports []ImportEntry @@ -70,18 +67,45 @@ type ImportsResult struct { Errors []error } -// ExportsResult is the result of gathering all the export statements from -// a source file, in case the language implementation explicitly exports certain files. -type ExportsResult struct { - // Symbols is an ordered map data structure where the keys are the symbols exported from - // the source file and the values are path from where they are declared. Symbols might - // be declared in a different path from where they are exported, for example: +// ExportSymbol represents a symbol that it's being exported. +// Exported symbols might be aliased in certain programming languages, for example, in JS: +// +// export { foo as bar } from './file' +// +// would export something that looks like this: +// +// ExportSymbol{ Original: "foo", Alias: "bar" } +type ExportSymbol struct { + // Original is the original value of the symbol. + Original string + // Alias is the alias under which the symbol is exported. + Alias string +} + +// ExportEntry represents an import statement in a programming language. +type ExportEntry struct { + // All means that all the symbols from AbsPath are exported. + // This is specially useful for languages that allow re-exporting all the symbols + // from a certain location, for example, in JS: + // + // export * from './file' + All bool + // Symbols are symbols that are exported from AbsPath. + Symbols []ExportSymbol + // AbsPath is absolute path from where they are exported, it might be from the same file or from another. + // Typically, this value will be the same as FileInfo.AbsPath, but in some language, you can re-export + // symbols that are declared in different source files, for example: // - // export { foo } from './bar' + // export function foo() {} // <- exports symbol `foo` that is declared in that same file. // - // the `foo` symbol is being exported from the current file, but it's declared on the - // `bar.ts` file. - Symbols *orderedmap.OrderedMap[string, string] + // export foo from './file' // <= exports symbol `foo` that is declared in another file. + AbsPath string +} + +// ExportsResult is the result of gathering all the export statements from a source file. +type ExportsResult struct { + // Exports is the list of ExportEntry in the source file. + Exports []ExportEntry // Errors are the non-fatal errors that occurred while parsing exports. These // might be rendered nicely in a UI. Errors []error @@ -96,5 +120,5 @@ type Language interface { ParseImports(file *FileInfo) (*ImportsResult, error) // ParseExports receives the file F parsed by the ParseFile method and gathers the exports that the file // F contains. - ParseExports(file *FileInfo) (*ExportsEntries, error) + ParseExports(file *FileInfo) (*ExportsResult, error) } diff --git a/internal/language/parser.go b/internal/language/parser.go index 0afe1b9..01c5da7 100644 --- a/internal/language/parser.go +++ b/internal/language/parser.go @@ -13,7 +13,7 @@ type Parser struct { // cache FileCache map[string]*FileInfo ImportsCache map[string]*ImportsResult - ExportsCache map[string]*ExportsResult + ExportsCache map[string]*ExportEntries } func NewParser(lang Language) *Parser { @@ -23,7 +23,7 @@ func NewParser(lang Language) *Parser { Exclude: nil, FileCache: make(map[string]*FileInfo), ImportsCache: make(map[string]*ImportsResult), - ExportsCache: make(map[string]*ExportsResult), + ExportsCache: make(map[string]*ExportEntries), } } diff --git a/internal/language/parser_test.go b/internal/language/parser_test.go index 997025c..61e1ff2 100644 --- a/internal/language/parser_test.go +++ b/internal/language/parser_test.go @@ -65,7 +65,7 @@ func TestParser_Deps(t *testing.T) { Name string Path string Imports map[string]*ImportsResult - Exports map[string]*ExportsEntries + Exports map[string]*ExportsResult ExpectedUnwrapped []string ExpectedWrapped []string }{ @@ -79,12 +79,12 @@ func TestParser_Deps(t *testing.T) { }, }, }, - Exports: map[string]*ExportsEntries{ + Exports: map[string]*ExportsResult{ "1": {}, "2": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}}, - Path: "2", + Symbols: []ExportSymbol{{Original: "Exported"}}, + AbsPath: "2", }}, }, }, @@ -101,17 +101,17 @@ func TestParser_Deps(t *testing.T) { Imports: map[string]*ImportsResult{ "1": {Imports: []ImportEntry{}}, }, - Exports: map[string]*ExportsEntries{ + Exports: map[string]*ExportsResult{ "1": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}}, - Path: "2", + Symbols: []ExportSymbol{{Original: "Exported"}}, + AbsPath: "2", }}, }, "2": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}}, - Path: "2", + Symbols: []ExportSymbol{{Original: "Exported"}}, + AbsPath: "2", }}, }, }, @@ -132,18 +132,18 @@ func TestParser_Deps(t *testing.T) { }, }, }, - Exports: map[string]*ExportsEntries{ + Exports: map[string]*ExportsResult{ "1": {}, "2": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}}, - Path: "3", + Symbols: []ExportSymbol{{Original: "Exported"}}, + AbsPath: "3", }}, }, "3": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}, {Original: "Another-one"}}, - Path: "3", + Symbols: []ExportSymbol{{Original: "Exported"}, {Original: "Another-one"}}, + AbsPath: "3", }}, }, }, @@ -162,35 +162,35 @@ func TestParser_Deps(t *testing.T) { Imports: []ImportEntry{}, }, }, - Exports: map[string]*ExportsEntries{ + Exports: map[string]*ExportsResult{ "1": { Exports: []ExportEntry{{ - All: true, - Path: "2", + All: true, + AbsPath: "2", }, { - Names: []ExportName{{Original: "Exported-3"}}, - Path: "3", + Symbols: []ExportSymbol{{Original: "Exported-3"}}, + AbsPath: "3", }}, }, "2": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}}, - Path: "4", + Symbols: []ExportSymbol{{Original: "Exported"}}, + AbsPath: "4", }, { - Names: []ExportName{{Original: "Exported-2"}}, - Path: "2", + Symbols: []ExportSymbol{{Original: "Exported-2"}}, + AbsPath: "2", }}, }, "3": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Another-one", Alias: "Exported-3"}}, - Path: "4", + Symbols: []ExportSymbol{{Original: "Another-one", Alias: "Exported-3"}}, + AbsPath: "4", }}, }, "4": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}, {Original: "Another-one"}}, - Path: "4", + Symbols: []ExportSymbol{{Original: "Exported"}, {Original: "Another-one"}}, + AbsPath: "4", }}, }, }, @@ -209,32 +209,32 @@ func TestParser_Deps(t *testing.T) { Imports: []ImportEntry{}, }, }, - Exports: map[string]*ExportsEntries{ + Exports: map[string]*ExportsResult{ "1": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported-3"}}, - Path: "3", + Symbols: []ExportSymbol{{Original: "Exported-3"}}, + AbsPath: "3", }}, }, "2": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}}, - Path: "4", + Symbols: []ExportSymbol{{Original: "Exported"}}, + AbsPath: "4", }, { - Names: []ExportName{{Original: "Exported-2"}}, - Path: "2", + Symbols: []ExportSymbol{{Original: "Exported-2"}}, + AbsPath: "2", }}, }, "3": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Another-one", Alias: "Exported-3"}}, - Path: "4", + Symbols: []ExportSymbol{{Original: "Another-one", Alias: "Exported-3"}}, + AbsPath: "4", }}, }, "4": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "Exported"}, {Original: "Another-one"}}, - Path: "4", + Symbols: []ExportSymbol{{Original: "Exported"}, {Original: "Another-one"}}, + AbsPath: "4", }}, }, }, @@ -287,7 +287,7 @@ func TestParser_DepsErrors(t *testing.T) { Name string Path string Imports map[string]*ImportsResult - Exports map[string]*ExportsEntries + Exports map[string]*ExportsResult ExpectedErrors []string }{ { @@ -301,12 +301,12 @@ func TestParser_DepsErrors(t *testing.T) { }, }}, }, - Exports: map[string]*ExportsEntries{ + Exports: map[string]*ExportsResult{ "1": {}, "2": { Exports: []ExportEntry{{ - Names: []ExportName{{Original: "bar"}}, - Path: "2", + Symbols: []ExportSymbol{{Original: "bar"}}, + AbsPath: "2", }}, }, }, diff --git a/internal/language/test_language.go b/internal/language/test_language.go index 5dc4aad..7eda6b4 100644 --- a/internal/language/test_language.go +++ b/internal/language/test_language.go @@ -11,7 +11,7 @@ type TestFileContent struct { type TestLanguage struct { imports map[string]*ImportsResult - exports map[string]*ExportsEntries + exports map[string]*ExportsResult } func (t *TestLanguage) testParser() *Parser { @@ -19,7 +19,7 @@ func (t *TestLanguage) testParser() *Parser { Lang: t, FileCache: map[string]*FileInfo{}, ImportsCache: map[string]*ImportsResult{}, - ExportsCache: map[string]*ExportsResult{}, + ExportsCache: map[string]*ExportEntries{}, } } @@ -42,7 +42,7 @@ func (t *TestLanguage) ParseImports(file *FileInfo) (*ImportsResult, error) { } } -func (t *TestLanguage) ParseExports(file *FileInfo) (*ExportsEntries, error) { +func (t *TestLanguage) ParseExports(file *FileInfo) (*ExportsResult, error) { time.Sleep(time.Millisecond) content := file.Content.(TestFileContent) if exports, ok := t.exports[content.Name]; ok { diff --git a/internal/python/exports.go b/internal/python/exports.go index d984705..fd9eb22 100644 --- a/internal/python/exports.go +++ b/internal/python/exports.go @@ -14,11 +14,11 @@ func (l *Language) handleFromImportForExport(imp *python_grammar.FromImport, fil } entry := language.ExportEntry{ - All: imp.All, - Path: filePath, + All: imp.All, + AbsPath: filePath, } for _, name := range imp.Names { - entry.Names = append(entry.Names, language.ExportName{ + entry.Symbols = append(entry.Symbols, language.ExportSymbol{ Original: name.Name, Alias: name.Alias, }) @@ -29,14 +29,14 @@ func (l *Language) handleFromImportForExport(imp *python_grammar.FromImport, fil case resolved.InitModule != nil: // nothing. case resolved.File != nil: - entry.Path = resolved.File.Path + entry.AbsPath = resolved.File.Path } return []language.ExportEntry{entry}, nil } //nolint:gocyclo -func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntries, error) { +func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsResult, error) { var exports []language.ExportEntry var errors []error @@ -47,13 +47,13 @@ func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntri continue case stmt.Import != nil && !stmt.Import.Indented && !l.cfg.IgnoreFromImportsAsExports: exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{ + Symbols: []language.ExportSymbol{ { Original: stmt.Import.Path[len(stmt.Import.Path)-1], Alias: stmt.Import.Alias, }, }, - Path: file.AbsPath, + AbsPath: file.AbsPath, }) case stmt.FromImport != nil && !stmt.FromImport.Indented && !l.cfg.IgnoreFromImportsAsExports: newExports, err := l.handleFromImportForExport(stmt.FromImport, file.AbsPath) @@ -65,38 +65,38 @@ func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntri case stmt.VariableUnpack != nil: entry := language.ExportEntry{ - Names: make([]language.ExportName, len(stmt.VariableUnpack.Names)), - Path: file.AbsPath, + Symbols: make([]language.ExportSymbol, len(stmt.VariableUnpack.Names)), + AbsPath: file.AbsPath, } for i, name := range stmt.VariableUnpack.Names { - entry.Names[i] = language.ExportName{Original: name} + entry.Symbols[i] = language.ExportSymbol{Original: name} } exports = append(exports, entry) case stmt.VariableAssign != nil: entry := language.ExportEntry{ - Names: make([]language.ExportName, len(stmt.VariableAssign.Names)), - Path: file.AbsPath, + Symbols: make([]language.ExportSymbol, len(stmt.VariableAssign.Names)), + AbsPath: file.AbsPath, } for i, name := range stmt.VariableAssign.Names { - entry.Names[i] = language.ExportName{Original: name} + entry.Symbols[i] = language.ExportSymbol{Original: name} } exports = append(exports, entry) case stmt.VariableTyping != nil: exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{{Original: stmt.VariableTyping.Name}}, - Path: file.AbsPath, + Symbols: []language.ExportSymbol{{Original: stmt.VariableTyping.Name}}, + AbsPath: file.AbsPath, }) case stmt.Function != nil: exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{{Original: stmt.Function.Name}}, - Path: file.AbsPath, + Symbols: []language.ExportSymbol{{Original: stmt.Function.Name}}, + AbsPath: file.AbsPath, }) case stmt.Class != nil: exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{{Original: stmt.Class.Name}}, - Path: file.AbsPath, + Symbols: []language.ExportSymbol{{Original: stmt.Class.Name}}, + AbsPath: file.AbsPath, }) } } - return &language.ExportsEntries{Exports: exports, Errors: errors}, nil + return &language.ExportsResult{Exports: exports, Errors: errors}, nil } diff --git a/internal/python/exports_test.go b/internal/python/exports_test.go index f8b9907..6a3d1cd 100644 --- a/internal/python/exports_test.go +++ b/internal/python/exports_test.go @@ -27,80 +27,80 @@ func TestLanguage_ParseExports(t *testing.T) { Entrypoint: "main.py", Expected: []language.ExportEntry{ { - Names: []language.ExportName{{Original: "foo", Alias: "foo_2"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "foo", Alias: "foo_2"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "foo", Alias: "foo_3"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "foo", Alias: "foo_3"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, //{ - // Names: []language.ExportName{{Original: "foo"}}, + // Names: []language.ExportSymbol{{Original: "foo"}}, // Path: filepath.Join(exportsTestFolder, "main.py"), // }, //{ - // Names: []language.ExportName{{Original: "folder", Alias: "foo"}}, + // Names: []language.ExportSymbol{{Original: "folder", Alias: "foo"}}, // Path: filepath.Join(exportsTestFolder, "main.py"), // }, //{ - // Names: []language.ExportName{{Original: "bar"}}, + // Names: []language.ExportSymbol{{Original: "bar"}}, // Path: filepath.Join(exportsTestFolder, "foo.py"), // }, //{ - // Names: []language.ExportName{{Original: "baz", Alias: "baz_2"}}, + // Names: []language.ExportSymbol{{Original: "baz", Alias: "baz_2"}}, // Path: filepath.Join(exportsTestFolder, "folder", "foo.py"), // }, { - Names: []language.ExportName{{Original: "a"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "a"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - All: true, - Path: filepath.Join(exportsTestFolder, "main.py"), + All: true, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "module"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "module"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "foo"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "foo"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "foo_1"}, {Original: "foo_2"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "foo_1"}, {Original: "foo_2"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "foo_3"}, {Original: "foo_4"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "foo_3"}, {Original: "foo_4"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "foo_5"}, {Original: "foo_6"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "foo_5"}, {Original: "foo_6"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "func"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "func"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, { - Names: []language.ExportName{{Original: "Class"}}, - Path: filepath.Join(exportsTestFolder, "main.py"), + Symbols: []language.ExportSymbol{{Original: "Class"}}, + AbsPath: filepath.Join(exportsTestFolder, "main.py"), }, //{ - // Names: []language.ExportName{{Original: "collections", Alias: "collections_abc"}}, + // Names: []language.ExportSymbol{{Original: "collections", Alias: "collections_abc"}}, // Path: filepath.Join(exportsTestFolder, "main.py"), // }, //{ - // Names: []language.ExportName{{Original: "collections", Alias: "collections_abc"}}, + // Names: []language.ExportSymbol{{Original: "collections", Alias: "collections_abc"}}, // Path: filepath.Join(exportsTestFolder, "main.py"), // }, { - Names: []language.ExportName{ + Symbols: []language.ExportSymbol{ {Original: "a"}, {Original: "b"}, {Original: "c"}, }, - Path: filepath.Join(exportsTestFolder, "foo.py"), + AbsPath: filepath.Join(exportsTestFolder, "foo.py"), }, }, }, diff --git a/internal/rust/exports.go b/internal/rust/exports.go index c3dd705..b5b97c8 100644 --- a/internal/rust/exports.go +++ b/internal/rust/exports.go @@ -7,7 +7,7 @@ import ( "github.com/gabotechs/dep-tree/internal/rust/rust_grammar" ) -func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntries, error) { +func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsResult, error) { exports := make([]language.ExportEntry, 0) var errors []error @@ -26,30 +26,30 @@ func (l *Language) ParseExports(file *language.FileInfo) (*language.ExportsEntri if use.All { exports = append(exports, language.ExportEntry{ - All: use.All, - Path: path, + All: use.All, + AbsPath: path, }) } else { exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{{Original: string(use.Name.Original), Alias: string(use.Name.Alias)}}, - Path: path, + Symbols: []language.ExportSymbol{{Original: string(use.Name.Original), Alias: string(use.Name.Alias)}}, + AbsPath: path, }) } } case stmt.Pub != nil: exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{{Original: string(stmt.Pub.Name)}}, - Path: file.AbsPath, + Symbols: []language.ExportSymbol{{Original: string(stmt.Pub.Name)}}, + AbsPath: file.AbsPath, }) case stmt.Mod != nil && stmt.Mod.Pub: exports = append(exports, language.ExportEntry{ - Names: []language.ExportName{{Original: string(stmt.Mod.Name)}}, - Path: file.AbsPath, + Symbols: []language.ExportSymbol{{Original: string(stmt.Mod.Name)}}, + AbsPath: file.AbsPath, }) } } - return &language.ExportsEntries{ + return &language.ExportsResult{ Exports: exports, Errors: errors, }, nil diff --git a/internal/rust/exports_test.go b/internal/rust/exports_test.go index 0ce0f09..7f71211 100644 --- a/internal/rust/exports_test.go +++ b/internal/rust/exports_test.go @@ -21,32 +21,32 @@ func TestLanguage_ParseExports(t *testing.T) { Name: "lib.rs", Expected: []language.ExportEntry{ { - Names: []language.ExportName{{Original: "div"}}, - Path: filepath.Join(absTestFolder, "src", "lib.rs"), + Symbols: []language.ExportSymbol{{Original: "div"}}, + AbsPath: filepath.Join(absTestFolder, "src", "lib.rs"), }, { - Names: []language.ExportName{{Original: "abs"}}, - Path: filepath.Join(absTestFolder, "src", "abs", "abs.rs"), + Symbols: []language.ExportSymbol{{Original: "abs"}}, + AbsPath: filepath.Join(absTestFolder, "src", "abs", "abs.rs"), }, { - Names: []language.ExportName{{Original: "div"}}, - Path: filepath.Join(absTestFolder, "src", "div", "mod.rs"), + Symbols: []language.ExportSymbol{{Original: "div"}}, + AbsPath: filepath.Join(absTestFolder, "src", "div", "mod.rs"), }, { - Names: []language.ExportName{{Original: "avg"}}, - Path: filepath.Join(absTestFolder, "src", "avg_2.rs"), + Symbols: []language.ExportSymbol{{Original: "avg"}}, + AbsPath: filepath.Join(absTestFolder, "src", "avg_2.rs"), }, { - Names: []language.ExportName{{Original: "sum"}}, - Path: filepath.Join(absTestFolder, "src", "lib.rs"), + Symbols: []language.ExportSymbol{{Original: "sum"}}, + AbsPath: filepath.Join(absTestFolder, "src", "lib.rs"), }, { - All: true, - Path: filepath.Join(absTestFolder, "src", "sum.rs"), + All: true, + AbsPath: filepath.Join(absTestFolder, "src", "sum.rs"), }, { - Names: []language.ExportName{{Original: "run"}}, - Path: filepath.Join(absTestFolder, "src", "lib.rs"), + Symbols: []language.ExportSymbol{{Original: "run"}}, + AbsPath: filepath.Join(absTestFolder, "src", "lib.rs"), }, }, },