Add setup.py scanning (#493)

analyzers/python/pip.go

@@ -8,12 +8,17 @@ import (
 func FromRequirements(reqs []pip.Requirement) []pkg.Import {
 	var imports []pkg.Import
 	for _, req := range reqs {
+		var revision string
+		// TODO: the backend isn't equipped to handle version ranges
+		if len(req.Constraints) > 0 {
+			revision = req.Constraints[0].Revision
+		}
 		imports = append(imports, pkg.Import{
 			Target: req.String(),
 			Resolved: pkg.ID{
-				Type:     pkg.Python,
-				Name:     req.Name,
-				Revision: req.Revision,
+				Type: pkg.Python,
+				Name: req.Name,
+				Revision: revision,
 			},
 		})
 	}

analyzers/python/python.go

@@ -84,7 +84,7 @@ func Discover(dir string, options map[string]interface{}) ([]module.Module, erro
 			return err
 		}
 
-		if !info.IsDir() && (info.Name() == "requirements.txt") {
+		if !info.IsDir() && (info.Name() == "requirements.txt" || info.Name() == "setup.py") {
 			moduleDir := filepath.Dir(filename)
 			_, ok := modules[moduleDir]
 			if ok {
@@ -160,18 +160,30 @@ func (a *Analyzer) Analyze() (graph.Deps, error) {
 	case "pipenv":
 		depGraph, err := a.Pipenv.Deps()
 		return depGraph, err
+	case "setuptools":
+		setupPyPath := filepath.Join(a.Module.Dir, "setup.py")
+		reqs, err := pip.FromSetupPy(setupPyPath)
+		if err != nil {
+			return graph.Deps{}, err
+		}
+		return requirementsToDeps(reqs), nil
 	case "requirements":
 		fallthrough
 	default:
-		reqs, err := pip.FromFile(a.requirementsFile(a.Module))
+		requirementsPath := a.requirementsFile(a.Module)
+		reqs, err := pip.FromFile(requirementsPath)
 		if err != nil {
 			return graph.Deps{}, err
 		}
-		imports := FromRequirements(reqs)
-		return graph.Deps{
-			Direct:     imports,
-			Transitive: fromImports(imports),
-		}, nil
+		return requirementsToDeps(reqs), nil
+	}
+}
+
+func requirementsToDeps(reqs []pip.Requirement) graph.Deps {
+	imports := FromRequirements(reqs)
+	return graph.Deps{
+		Direct:     imports,
+		Transitive: fromImports(imports),
 	}
 }
 

buildtools/pip/pip.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"unicode"
 
 	"github.com/apex/log"
 
@@ -12,21 +13,37 @@ import (
 	"github.com/fossas/fossa-cli/files"
 )
 
-// TODO: add a Python sidecar that evaluates `setup.py`.
-
 type Pip struct {
 	Cmd       string
 	PythonCmd string
 }
 
-type Requirement struct {
+// Structure for deserializing the output of `pip list`
+type listRequirement struct {
 	Name     string `json:"name"`
 	Revision string `json:"version"`
+}
+
+type Requirement struct {
+	Name        string
+	Constraints []Constraint
+}
+
+type Constraint struct {
+	Revision string
 	Operator string
 }
 
+func (c Constraint) String() string {
+	return c.Operator + c.Revision
+}
+
 func (r Requirement) String() string {
-	return r.Name + r.Operator + r.Revision
+	var constraintStrings []string
+	for _, constraint := range r.Constraints {
+		constraintStrings = append(constraintStrings, constraint.String())
+	}
+	return r.Name + strings.Join(constraintStrings, ",")
 }
 
 func (p *Pip) Install(requirementsFilename string) *errors.Error {
@@ -56,9 +73,8 @@ func (p *Pip) List() ([]Requirement, *errors.Error) {
 			Link:            "https://github.com/fossas/fossa-cli/blob/master/docs/integrations/python.md#strategy-string",
 		}
 	}
-
-	var reqs []Requirement
-	err = json.Unmarshal([]byte(stdout), &reqs)
+	var listReqs []listRequirement
+	err = json.Unmarshal([]byte(stdout), &listReqs)
 	if err != nil {
 		return nil, &errors.Error{
 			Cause:           err,
@@ -67,6 +83,78 @@ func (p *Pip) List() ([]Requirement, *errors.Error) {
 			Link:            "https://pip.pypa.io/en/stable/reference/pip_list",
 		}
 	}
+
+	var reqs []Requirement
+	for _, listReq := range listReqs {
+		reqs = append(reqs, Requirement{
+			Name:        listReq.Name,
+			Constraints: []Constraint{{Revision: listReq.Revision}},
+		})
+	}
+	return reqs, nil
+}
+
+const requiresField = "install_requires"
+
+func FromSetupPy(filename string) ([]Requirement, *errors.Error) {
+	raw, err := files.Read(filename)
+	if err != nil {
+		return nil, &errors.Error{
+			Cause:           err,
+			Type:            errors.User,
+			Troubleshooting: fmt.Sprintf("Ensure that `%s` exists.", filename),
+			Link:            "https://github.com/fossas/fossa-cli/blob/master/docs/integrations/python.md#analysis",
+		}
+	}
+
+	contents := string(raw)
+
+	reqStart := strings.Index(contents, requiresField)
+	if reqStart == -1 {
+		return nil, nil
+	}
+
+	contents = contents[(reqStart + len(requiresField)):]
+
+	openBracket := strings.Index(contents, "[")
+	if openBracket == -1 {
+		return nil, &errors.Error{
+			Type:    errors.Exec,
+			Message: "Failed to parse setup.py: expected to find '[' after \"" + requiresField + "\"",
+		}
+	}
+
+	contents = contents[openBracket+1:]
+
+	endBracket := strings.Index(contents, "]")
+	if endBracket == -1 {
+		return nil, &errors.Error{
+			Type:    errors.Exec,
+			Message: "Failed to parse setup.py: expected to find ']' after \"" + requiresField + "\"",
+		}
+	}
+
+	var reqs []Requirement
+	for {
+		endBracket := strings.Index(contents, "]")
+		openQuote := strings.Index(contents, "'")
+		if openQuote == -1 || openQuote > endBracket {
+			break
+		}
+
+		contents = contents[openQuote+1:]
+		closeQuote := strings.Index(contents, "'")
+		if closeQuote == -1 {
+			// NB: silently failing here instead of throwing an error
+			break
+		}
+
+		depString := contents[:closeQuote]
+		contents = contents[closeQuote+1:]
+
+		reqs = append(reqs, parseRequirement(depString))
+	}
+
 	return reqs, nil
 }
 
@@ -87,36 +175,62 @@ func FromFile(filename string) ([]Requirement, *errors.Error) {
 		// Remove all line comments and whitespace.
 		commentSplit := strings.Split(line, "#")
 		trimmed := strings.TrimSpace(commentSplit[0])
-		trimmed = strings.ReplaceAll(trimmed, " ", "")
 		if strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "-") || trimmed == "" {
 			continue
 		}
 
 		log.WithField("line", line).Debug("parsing line")
-		// See https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
-		// and https://pip.pypa.io/en/stable/reference/pip_install/#pip-install-examples
-		matched := false
-		operators := []string{"===", "<=", ">=", "==", ">", "<", "!=", "~="}
-		for _, op := range operators {
-			sections := strings.Split(trimmed, op)
-			if len(sections) == 2 {
-				reqs = append(reqs, Requirement{
-					Name:     checkForExtra(sections[0]),
-					Revision: sections[1],
-					Operator: op,
-				})
-				matched = true
-				break
-			}
+		req := parseRequirement(trimmed)
+		req.Name = checkForExtra(req.Name)
+		reqs = append(reqs, req)
+	}
+
+	return reqs, nil
+}
+
+func parseRequirement(raw string) Requirement {
+	// See https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
+	// and https://pip.pypa.io/en/stable/reference/pip_install/#pip-install-examples
+	trimmed := strings.ReplaceAll(raw, " ", "")
+
+	constraintsStart := -1
+	for ix, char := range trimmed {
+		// operators start with these characters
+		if char == '~' || char == '=' || char == '>' || char == '<' || char == '!' {
+			constraintsStart = ix
+			break
 		}
-		if !matched {
-			reqs = append(reqs, Requirement{
-				Name: checkForExtra(trimmed),
-			})
+	}
+
+	name := trimmed
+	var constraints []Constraint
+
+	if constraintsStart != -1 {
+		name = trimmed[:constraintsStart]
+		constraintsRaw := strings.Split(trimmed[constraintsStart:], ",")
+		for _, constraintRaw := range constraintsRaw {
+			constraints = append(constraints, parseConstraint(constraintRaw))
 		}
 	}
 
-	return reqs, nil
+	return Requirement{
+		Name:        name,
+		Constraints: constraints,
+	}
+}
+
+func parseConstraint(raw string) Constraint {
+	for ix, char := range raw {
+		if unicode.IsDigit(char) {
+			return Constraint{
+				Revision: raw[ix:],
+				Operator: raw[:ix],
+			}
+		}
+	}
+	return Constraint{
+		Revision: raw,
+	}
 }
 
 // https://www.python.org/dev/peps/pep-0508/#extras

buildtools/pip/pip_test.go

@@ -12,14 +12,23 @@ func TestFromFile(t *testing.T) {
 	reqs, err := pip.FromFile("testdata/requirements.txt")
 	assert.Nil(t, err)
 	assert.Equal(t, 8, len(reqs))
-	assert.Contains(t, reqs, pip.Requirement{Name: "simple", Revision: "1.0.0", Operator: "=="})
-	assert.Contains(t, reqs, pip.Requirement{Name: "extra", Revision: "2.0.0", Operator: "=="})
+	assert.Contains(t, reqs, pip.Requirement{Name: "simple", Constraints: []pip.Constraint{{Revision: "1.0.0", Operator: "=="}}})
+	assert.Contains(t, reqs, pip.Requirement{Name: "extra", Constraints: []pip.Constraint{{Revision: "2.0.0", Operator: "=="}}})
 	assert.Contains(t, reqs, pip.Requirement{Name: "latest"})
 	assert.Contains(t, reqs, pip.Requirement{Name: "latestExtra"})
-	assert.Contains(t, reqs, pip.Requirement{Name: "notEqualOp", Revision: "3.0.0", Operator: ">="})
-	assert.Contains(t, reqs, pip.Requirement{Name: "comment-version", Revision: "2.0.0", Operator: "==="})
+	assert.Contains(t, reqs, pip.Requirement{Name: "notEqualOp", Constraints: []pip.Constraint{{Revision: "3.0.0", Operator: ">="}}})
+	assert.Contains(t, reqs, pip.Requirement{Name: "comment-version", Constraints: []pip.Constraint{{Revision: "2.0.0", Operator: "==="}}})
 	assert.Contains(t, reqs, pip.Requirement{Name: "comment"})
-	assert.Contains(t, reqs, pip.Requirement{Name: "tilde", Revision: "2.0.0", Operator: "~="})
+	assert.Contains(t, reqs, pip.Requirement{Name: "tilde", Constraints: []pip.Constraint{{Revision: "2.0.0", Operator: "~="}}})
 	assert.NotContains(t, reqs, pip.Requirement{Name: "-r other-requirements.txt"})
 	assert.NotContains(t, reqs, pip.Requirement{Name: "--option test-option"})
 }
+
+func TestFromSetupPy(t *testing.T) {
+	reqs, err := pip.FromSetupPy("testdata/setup.py")
+	assert.Nil(t, err)
+	assert.Contains(t, reqs, pip.Requirement{Name: "simple", Constraints: []pip.Constraint{{Revision: "1.0.0", Operator: "=="}}})
+	assert.Contains(t, reqs, pip.Requirement{Name: "gteq", Constraints: []pip.Constraint{{Revision: "2.0.0", Operator: ">="}}})
+	assert.Contains(t, reqs, pip.Requirement{Name: "sameline", Constraints: []pip.Constraint{{Revision: "1.0.0", Operator: "=="}}})
+	assert.Contains(t, reqs, pip.Requirement{Name: "latest", Constraints: nil})
+}

buildtools/pip/testdata/setup.py

@@ -0,0 +1,11 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='sampleproject',
+    version='1.3.0',
+    description='A sample Python project',
+    install_requires=['simple==1.0.0',
+                      'gteq>=2.0.0', 'sameline==1.0.0',
+                      'latest'
+                      ],
+)

docs/integrations/python.md

@@ -5,6 +5,7 @@
 Python support relies on the presence of one of the following:
 
 - A `requirements.txt` file.
+- A `setup.py` file.
 - `pip`, in order to retrieve a list of installed dependencies.
 - Pipenv, used to manage a projects environment and dependencies.
 
@@ -44,7 +45,8 @@ analyze:
 #### `strategy: <string>`
 
 Manually specify the python analysis strategy to be used. Supported options:
-- `requirements`: Parse `requirements.txt` to find all dependencies used. 
+- `requirements`: Parse `requirements.txt` to find all dependencies used.
+- `setuptools`: Parse `setup.py` to find dependencies.
 - `pip`: Run `pip list --format=json` to find all dependencies in the current environment. `pip` over report the dependencies used if your environment is used to build multiple python projects.
 - `deptree`: Run a custom python script to retrieve the dependency tree from pip. This provides similar information to `pip` with enough resolution to create a dependency tree.
 - `pipenv`: Run `pipenv graph --json=tree` which returns the dependency graph of a project managed by Pipenv.
@@ -62,8 +64,9 @@ Example:
 
 ## Analysis
 
-The analysis strategy selected determines how analysis is completed for the Python analyzer. By default the fossa-cli will analyze a requirements.txt file to determine dependencies. Benefits and limitations of strategies are listed below.
+The analysis strategy selected determines how analysis is completed for the Python analyzer. By default the fossa-cli will analyze a requirements.txt to determine dependencies. Benefits and limitations of strategies are listed below.
 
-- `requirements`: This strategy is the most basic but provides an accurate representation of all dependencies inside of `requirements.txt`. The limitations with this method include not picking up transitive dependencies unless they are explicitly added to the file.
+- `requirements`: This strategy provides an accurate representation of all dependencies inside of `requirements.txt`. The limitations with this method include not picking up transitive dependencies unless they are explicitly added to the file.
+- `setuptools`: This strategy is the most basic, and provides an incomplete representation of dependencies used by the project. We may incorrectly identify specific project versions, and transitive dependencies won't be discovered.
 - `pip` & `deptree`: These strategies can accurately provide a dependency graph, however they analyze all dependencies managed by pip, not just those in the project. If your project is built in a CI environment where all pip installed dependencies are used, then this strategy would be effective. If you are on a local development machine then this strategy can over report dependencies.
 - `pipenv`: This is the most reliable analysis strategy but requires your project to use Pipenv as its environment and package manager.