refactor(sbt): Use graph XML parsing instead of output parsing (#198)

analyzers/scala/scala.go

@@ -156,45 +156,8 @@ func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
 		return m, err
 	}
 
-	// Set direct dependencies.
-	var i []pkg.Import
-	for _, dep := range imports {
-		i = append(i, pkg.Import{
-			Resolved: pkg.ID{
-				Type:     pkg.Scala,
-				Name:     dep.Name,
-				Revision: dep.Version,
-			},
-		})
-	}
-
-	// Set transitive dependencies.
-	g := make(map[pkg.ID]pkg.Package)
-	for parent, children := range graph {
-		id := pkg.ID{
-			Type:     pkg.Scala,
-			Name:     parent.Name,
-			Revision: parent.Version,
-		}
-		var imports []pkg.Import
-		for _, child := range children {
-			imports = append(imports, pkg.Import{
-				Target: child.Version,
-				Resolved: pkg.ID{
-					Type:     pkg.Scala,
-					Name:     child.Name,
-					Revision: child.Version,
-				},
-			})
-		}
-		g[id] = pkg.Package{
-			ID:      id,
-			Imports: imports,
-		}
-	}
-
-	m.Imports = i
-	m.Deps = g
+	m.Imports = imports
+	m.Deps = graph
 	return m, nil
 }
 

buildtools/sbt/parse.go

@@ -0,0 +1,106 @@
+package sbt
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/fossas/fossa-cli/log"
+	"github.com/fossas/fossa-cli/pkg"
+)
+
+type GraphML struct {
+	Graph Graph `xml:"graph"`
+}
+
+type Graph struct {
+	Nodes []Node `xml:"node"`
+	Edges []Edge `xml:"edge"`
+}
+
+type Node struct {
+	ID string `xml:"id,attr"`
+}
+
+type Edge struct {
+	Source string `xml:"source,attr"`
+	Target string `xml:"target,attr"`
+}
+
+func ParseDependencyGraph(graph Graph, evicted string) (pkg.Imports, pkg.Deps, error) {
+	log.Logger.Debugf("%#v %#v", graph, evicted)
+
+	replacements := ParseEvicted(evicted)
+
+	deps := make(map[pkg.ID]pkg.Imports)
+	for _, edge := range graph.Edges {
+		source := ParsePackageID(edge.Source)
+		target := ParsePackageID(edge.Target)
+
+		log.Logger.Debugf("source: %#v", source)
+
+		_, ok := deps[source]
+		if !ok {
+			deps[source] = pkg.Imports{}
+		}
+
+		replacement, ok := replacements[target]
+		if ok {
+			target = replacement
+		}
+
+		deps[source] = append(deps[source], pkg.Import{
+			Target:   target.String(),
+			Resolved: target,
+		})
+	}
+
+	pkgs := make(pkg.Deps)
+	for id, imports := range deps {
+		pkgs[id] = pkg.Package{
+			ID:      id,
+			Imports: imports,
+		}
+	}
+
+	root := ParsePackageID(graph.Nodes[0].ID)
+	imports := deps[root]
+	delete(pkgs, root)
+
+	return imports, pkgs, nil
+}
+
+func ParseEvicted(evicted string) map[pkg.ID]pkg.ID {
+	replacements := make(map[pkg.ID]pkg.ID)
+	r := regexp.MustCompile("^\\[info\\] 	\\* (.*?) is selected over (.*?)$")
+	for _, line := range strings.Split(evicted, "\n") {
+		matches := r.FindStringSubmatch(line)
+		if matches != nil {
+			target := ParsePackageID(matches[1])
+			if strings.HasPrefix(matches[2], "{") && strings.HasSuffix(matches[2], "}") {
+				// Handle multiple versions.
+				versions := strings.Split(strings.Trim(matches[2], "{}"), ", ")
+				for _, version := range versions {
+					source := target
+					source.Revision = version
+					replacements[source] = target
+				}
+			} else {
+				// Handle single version.
+				source := target
+				source.Revision = matches[2]
+				replacements[source] = target
+			}
+		}
+	}
+	return replacements
+}
+
+func ParsePackageID(packageID string) pkg.ID {
+	r := regexp.MustCompile("([^:\\s]+):([^:\\s]+):([^:\\s]+).*")
+	matches := r.FindStringSubmatch(packageID)
+	return pkg.ID{
+		Type:     pkg.Scala,
+		Name:     matches[1] + ":" + matches[2],
+		Revision: matches[3],
+	}
+}

buildtools/sbt/sbt.go

@@ -7,7 +7,9 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/fossas/fossa-cli/exec"
+	"github.com/fossas/fossa-cli/files"
 	"github.com/fossas/fossa-cli/log"
+	"github.com/fossas/fossa-cli/pkg"
 )
 
 type SBT struct {
@@ -64,24 +66,48 @@ func (s *SBT) Projects(dir string) ([]string, error) {
 	return projects, nil
 }
 
-//go:generate bash -c "genny -in=$GOPATH/src/github.com/fossas/fossa-cli/graph/readtree.go gen 'Generic=Dependency' | sed -e 's/package graph/package sbt/' > readtree_generated.go"
-
 type Dependency struct {
 	Name    string
 	Version string
 }
 
-func (s *SBT) DependencyTree(dir, project, configuration string) ([]Dependency, map[Dependency][]Dependency, error) {
+func (s *SBT) DependencyTree(dir, project, configuration string) (pkg.Imports, pkg.Deps, error) {
 	output, _, err := exec.Run(exec.Cmd{
 		Dir:  dir,
 		Name: s.Bin,
-		Argv: []string{"-no-colors", Task(project, configuration, "dependencyTree")},
+		Argv: []string{"-no-colors", Task(project, configuration, "dependencyGraphMl")},
+	})
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not get dependency graph from SBT")
+	}
+
+	file := ""
+	r := regexp.MustCompile("^\\[info\\] Wrote dependency graph to '(.*?)'$")
+	for _, line := range strings.Split(output, "\n") {
+		matches := r.FindStringSubmatch(line)
+		if matches != nil {
+			file = matches[1]
+		}
+	}
+	log.Logger.Debugf("file: %#v", file)
+
+	var root GraphML
+	err = files.ReadXML(&root, file)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not parse SBT dependency graph")
+	}
+	log.Logger.Debugf("graph: %#v", root)
+
+	evicted, _, err := exec.Run(exec.Cmd{
+		Dir:  dir,
+		Name: s.Bin,
+		Argv: []string{"-no-colors", Task(project, configuration, "evicted")},
 	})
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "could not get dependency tree from SBT")
+		return nil, nil, errors.Wrap(err, "could not get version conflict resolutions from SBT")
 	}
 
-	return ParseDependencyTree(output, project == "root" || project == "")
+	return ParseDependencyGraph(root.Graph, evicted)
 }
 
 func (s *SBT) DependencyList(dir, project, configuration string) (string, error) {
@@ -107,61 +133,6 @@ func (s *SBT) DependencyList(dir, project, configuration string) (string, error)
 	return strings.Join(depLines, "\n"), err
 }
 
-func ParseDependencyTree(output string, rootBuild bool) ([]Dependency, map[Dependency][]Dependency, error) {
-	// Filter lines to only include dependency tree.
-	spacerRegex := regexp.MustCompile("^\\[info\\] ([ `+\\\\|-]*)(\\s*?)$")
-	var depLines []string
-	for _, line := range strings.Split(output, "\n") {
-		if FilterLine(line) && !spacerRegex.MatchString(line) {
-			log.Logger.Debugf("Matched line: %#v", line)
-			depLines = append(depLines, line)
-		} else {
-			log.Logger.Debugf("Ignoring line: %#v", line)
-		}
-	}
-
-	// Non-root builds only have one sub-project, so we collapse the first layer
-	// of imports.
-	if !rootBuild {
-		// Remove the sub-project line.
-		depLines = depLines[1:]
-	}
-
-	depRegex := regexp.MustCompile("^\\[info\\] ([ `+\\\\|-]*)([^ `+\\\\|-].+)$")
-	locatorRegex := regexp.MustCompile("([^:\\s]+):([^:\\s]+):([^:\\s]+).*")
-	return ReadDependencyTree(depLines, func(line string) (int, Dependency, error) {
-		log.Logger.Debugf("Parsing line: %#v\n", line)
-
-		// Match for context
-		depMatches := depRegex.FindStringSubmatch(line)
-		log.Logger.Debugf("Dep matches: %#v\n", depMatches)
-		depth := len(depMatches[1])
-
-		// SBT quirk: the indentation from level 1 to level 2 is 4 spaces, but all others are 2 spaces
-		if depth >= 4 {
-			depth -= 2
-		}
-		// Handle non-root builds.
-		if !rootBuild {
-			depth -= 2
-		}
-
-		if depth%2 != 0 {
-			// Sanity check
-			log.Logger.Panicf("Bad depth: %#v %s %#v", depth, line, depMatches)
-		}
-		// Parse locator
-		locatorMatches := locatorRegex.FindStringSubmatch(depMatches[2])
-		log.Logger.Debugf("Locator matches: %#v\n", locatorMatches)
-		dep := Dependency{
-			Name:    locatorMatches[1] + ":" + locatorMatches[2],
-			Version: locatorMatches[3],
-		}
-
-		return depth / 2, dep, nil
-	})
-}
-
 func FilterLine(line string) bool {
 	trimmed := strings.TrimSpace(line)
 	if !strings.HasPrefix(trimmed, "[info] ") {

buildtools/sbt/sbt_test.go

@@ -5,24 +5,27 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/fossas/fossa-cli/files"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/fossas/fossa-cli/buildtools/sbt"
-	"github.com/fossas/fossa-cli/log"
 )
 
 func TestSanityCheckParseDependencyTree(t *testing.T) {
-	// This is a hack so that logging from this test does not swamp all other log
-	// output.
-	log.Init(false, false)
+	var graph sbt.GraphML
+	err := files.ReadXML(&graph, filepath.Join("testdata", "sbt_dependencygraphml-prisma.xml"))
+	assert.NoError(t, err)
 
-	fixture, err := ioutil.ReadFile(filepath.Join("testdata", "sbt_dependencytree_nocolor-prisma-stdout"))
+	evicted, err := ioutil.ReadFile(filepath.Join("testdata", "sbt_evicted_nocolor-prisma"))
 	assert.NoError(t, err)
 
-	imports, graphs, err := sbt.ParseDependencyTree(string(fixture), true)
+	_, pkgs, err := sbt.ParseDependencyGraph(graph.Graph, string(evicted))
 	assert.NoError(t, err)
-	t.Log(imports)
-	t.Log(graphs)
+
+	for id, pkg := range pkgs {
+		assert.Equal(t, id, pkg.ID)
+	}
 }
 
 func TestFilterLines(t *testing.T) {