GO LIST JSON? WHY NOT!? (#150)

💥

.gitignore

@@ -11,3 +11,6 @@ dist/
 
 # ci config for local ci build
 .circleci/local-config.yml
+
+# VS Code
+.vscode

README.md

@@ -30,11 +30,15 @@
 
 ### Usage
 
+`nancy` currently works for projects that use `dep` or `go mod` for dependencies.
+
 ```
  ~ > nancy
 Usage:
         go list -m all | nancy [options]
         go list -m all | nancy iq [options]
+        go list -json -m all | nancy [options]
+        go list -json -m all | nancy iq [options]
         nancy config
         nancy [options] </path/to/Gopkg.lock>
         nancy [options] </path/to/go.sum>
@@ -69,6 +73,7 @@ Options:
 $ > nancy iq
 Usage:
         go list -m all | nancy iq [options]
+        go list -json -m all | nancy iq [options]
 
 Options:
   -application string
@@ -90,7 +95,21 @@ Options:
         Set log level to Trace
 ```
 
-`nancy` currently works for projects that use `dep` or `go mod` for dependencies.
+#### What is the best usage of Nancy?
+
+There are four different ways to use Nancy:
+
+Preferred (will be around in Nancy 1.0.0)
+- `go list -json -m all | nancy`
+- `nancy /path/to/Gopkg.lock`
+
+Unpreferred (will be deprecated in Nancy 1.0.0)
+- `go list -m all | nancy`
+- `nancy /path/to/go.sum`
+
+The reasons for this are myriad, and are:
+- `go.sum` files are not lockfiles, they can contain many entries that your project no longer uses, and are thus, not totally reliable for a source of "what is my project using right now". We have yet to remove this functionality, mostly so we can gracefully remove it when Nancy 1.0.0 arrives (soon!)
+- `go list -m all` as a command is great! However, it outputs everything as plain text. Using `-json` in combo with it gives us a data structure that we can parse easily, and will allow us to implement some cool new features over time!
 
 #### Homebrew usage
 

configuration/parse.go

@@ -84,6 +84,7 @@ func ParseIQ(args []string) (config IqConfiguration, err error) {
 	flag.Usage = func() {
 		_, _ = fmt.Fprintf(os.Stderr, `Usage:
 	go list -m all | nancy iq [options]
+	go list -json -m all | nancy iq [options]
 			
 Options:
 `)
@@ -165,6 +166,8 @@ func Parse(args []string) (Configuration, error) {
 		_, _ = fmt.Fprintf(os.Stderr, `Usage:
 	go list -m all | nancy [options]
 	go list -m all | nancy iq [options]
+	go list -json -m all | nancy [options]
+	go list -json -m all | nancy iq [options]
 	nancy config
 	nancy [options] </path/to/Gopkg.lock>
 	nancy [options] </path/to/go.sum>

main.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"bufio"
 	"errors"
 	"flag"
 	"fmt"
@@ -246,10 +245,13 @@ func doStdInAndParse(config configuration.Configuration) (err error) {
 	LogLady.Info("Instantiating go.mod package")
 
 	mod := packages.Mod{}
-	scanner := bufio.NewScanner(os.Stdin)
 
 	LogLady.Info("Beginning to parse StdIn")
-	mod.ProjectList, _ = parse.GoList(scanner)
+	mod.ProjectList, err = parse.GoListAgnostic(os.Stdin)
+	if err != nil {
+		LogLady.Error(err)
+		return
+	}
 	LogLady.WithFields(logrus.Fields{
 		"projectList": mod.ProjectList,
 	}).Debug("Obtained project list")
@@ -285,10 +287,13 @@ func doStdInAndParseForIQ(config configuration.IqConfiguration) (err error) {
 	LogLady.Info("Instantiating go.mod package")
 
 	mod := packages.Mod{}
-	scanner := bufio.NewScanner(os.Stdin)
 
 	LogLady.Info("Beginning to parse StdIn")
-	mod.ProjectList, _ = parse.GoList(scanner)
+	mod.ProjectList, err = parse.GoListAgnostic(os.Stdin)
+	if err != nil {
+		LogLady.Error(err)
+		return
+	}
 	LogLady.WithFields(logrus.Fields{
 		"projectList": mod.ProjectList,
 	}).Debug("Obtained project list")

parse/parse.go

@@ -18,6 +18,10 @@ package parse
 
 import (
 	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"os"
 	"strings"
 
@@ -39,6 +43,68 @@ func GoList(stdIn *bufio.Scanner) (deps types.ProjectList, err error) {
 	return deps, nil
 }
 
+// GoListAgnostic will take a io.Reader that is likely the os.StdIn, try to parse it as
+// if `go list -json -m all` was ran, and then try to reparse it as if `go list -m all`
+// was ran instead. It returns either an error, or a deps of types.ProjectList
+func GoListAgnostic(stdIn io.Reader) (deps types.ProjectList, err error) {
+	// stdIn should never be massive, so taking this approach over reading from a stream
+	// multiple times
+	johnnyFiveNeedInput, err := ioutil.ReadAll(stdIn)
+	if err != nil {
+		return
+	}
+	decoder := json.NewDecoder(strings.NewReader(string(johnnyFiveNeedInput)))
+
+	for {
+		var mod types.GoListModule
+
+		err = decoder.Decode(&mod)
+		if err == io.EOF {
+			err = nil
+			break
+		}
+		if err != nil {
+			break
+		}
+
+		project, err := modToProjectList(mod)
+		if _, ok := err.(*NoVersionError); ok {
+			continue
+		}
+		deps.Projects = append(deps.Projects, project)
+	}
+
+	if err != nil {
+		err = nil
+		scanner := bufio.NewScanner(strings.NewReader(string(johnnyFiveNeedInput)))
+		deps, err = GoList(scanner)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func modToProjectList(mod types.GoListModule) (dep types.Projects, err error) {
+	if mod.Replace != nil {
+		if mod.Replace.Version == "" {
+			err = &NoVersionError{err: fmt.Errorf("No version found for mod")}
+			return
+		}
+		dep.Name = mod.Replace.Path
+		dep.Version = mod.Replace.Version
+		return
+	}
+	if mod.Version == "" {
+		err = &NoVersionError{err: fmt.Errorf("No version found for mod")}
+		return
+	}
+	dep.Name = mod.Path
+	dep.Version = mod.Version
+	return
+}
+
 // GoSum parses the go.sum file and returns an error if unsuccessful
 func GoSum(path string) (deps types.ProjectList, err error) {
 	file, err := os.Open(path)
@@ -59,6 +125,19 @@ func parseSpaceSeparatedDependency(scanner *bufio.Scanner, deps *types.ProjectLi
 	text := scanner.Text()
 	s := strings.Split(text, " ")
 	if criteria(s) {
-		deps.Projects = append(deps.Projects, types.Projects{Name: s[0], Version: s[1]})
+		if len(s) > 3 {
+			fmt.Print(s)
+			deps.Projects = append(deps.Projects, types.Projects{Name: s[0], Version: s[4]})
+		} else {
+			deps.Projects = append(deps.Projects, types.Projects{Name: s[0], Version: s[1]})
+		}
 	}
 }
+
+type NoVersionError struct {
+	err error
+}
+
+func (n *NoVersionError) Error() string {
+	return n.err.Error()
+}

parse/parse_test.go

@@ -18,6 +18,7 @@ package parse
 
 import (
 	"bufio"
+	"os"
 	"strings"
 	"testing"
 )
@@ -33,6 +34,66 @@ func TestGoSum(t *testing.T) {
 	}
 }
 
+func TestGoListAgnostic(t *testing.T) {
+	goListFile, err := os.Open("testdata/golist.out")
+	if err != nil {
+		t.Error(err)
+	}
+
+	deps, err := GoListAgnostic(goListFile)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(deps.Projects) != 48 {
+		t.Errorf("Unsuccessfully parsed go list -m all output, 48 dependencies were expected, but %d encountered", len(deps.Projects))
+	}
+
+	goListJSONFile, err := os.Open("testdata/golistjson.out")
+	if err != nil {
+		t.Error(err)
+	}
+
+	deps, err = GoListAgnostic(goListJSONFile)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(deps.Projects) != 48 {
+		t.Errorf("Unsuccessfully parsed go list -json -m all output, 48 dependencies were expected, but %d encountered", len(deps.Projects))
+	}
+
+	goListReplaceFile, err := os.Open("testdata/golistreplace.out")
+	if err != nil {
+		t.Error(err)
+	}
+
+	deps, err = GoListAgnostic(goListReplaceFile)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(deps.Projects) != 1 {
+		t.Errorf("Unsuccessfully parsed go list -m all output, 1 dependency was expected, but %d encountered", len(deps.Projects))
+	}
+	if deps.Projects[0].Version != "v1.4.2" {
+		t.Errorf("Version expected to be v1.4.2, but encountered %s", deps.Projects[0].Version)
+	}
+
+	goListJSONReplaceFile, err := os.Open("testdata/golistjsonreplace.out")
+	if err != nil {
+		t.Error(err)
+	}
+
+	deps, err = GoListAgnostic(goListJSONReplaceFile)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(deps.Projects) != 134 {
+		t.Errorf("Unsuccessfully parsed go list -m all output, 134 dependencies were expected, but %d encountered", len(deps.Projects))
+	}
+	if deps.Projects[0].Version != "v1.4.2" {
+		t.Errorf("Version expected to be v1.4.2, but encountered %s", deps.Projects[0].Version)
+	}
+}
+
 func TestGoListAll(t *testing.T) {
 	goListMAllOutput := `github.com/sonatype-nexus-community/nancy
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7

parse/testdata/golist.out

@@ -0,0 +1,49 @@
+github.com/sonatype-nexus-community/nancy
+github.com/BurntSushi/toml v0.3.1
+github.com/Flaque/filet v0.0.0-20190209224823-fc4d33cfcf93
+github.com/Masterminds/semver v0.0.0-20180403130225-3c92f33da7a8
+github.com/Masterminds/vcs v1.13.1
+github.com/armon/go-radix v1.0.0
+github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
+github.com/beevik/etree v1.1.0
+github.com/boltdb/bolt v1.3.1
+github.com/common-nighthawk/go-figure v0.0.0-20190529165535-67e0ed34491a
+github.com/davecgh/go-spew v1.1.1
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
+github.com/go-openapi/errors v0.19.0
+github.com/go-openapi/strfmt v0.19.0
+github.com/golang/dep v0.5.4
+github.com/golang/protobuf v1.2.0
+github.com/google/go-cmp v0.4.0
+github.com/google/uuid v1.1.1
+github.com/jarcoal/httpmock v1.0.4
+github.com/jedib0t/go-pretty/v6 v6.0.2
+github.com/jmank88/nuts v0.3.0
+github.com/konsorten/go-windows-terminal-sequences v1.0.2
+github.com/kr/pretty v0.1.0
+github.com/kr/pty v1.1.1
+github.com/kr/text v0.1.0
+github.com/logrusorgru/aurora v0.0.0-20190803045625-94edacc10f9b
+github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983
+github.com/mattn/go-runewidth v0.0.9
+github.com/mitchellh/mapstructure v1.1.2
+github.com/nightlyone/lockfile v0.0.0-20180618180623-0ad87eef1443
+github.com/package-url/packageurl-go v0.1.0
+github.com/pelletier/go-toml v1.4.0
+github.com/pkg/errors v0.8.0
+github.com/pkg/profile v1.2.1
+github.com/pmezard/go-difflib v1.0.0
+github.com/recoilme/pudge v1.0.3
+github.com/sdboyer/constext v0.0.0-20170321163424-836a14457353
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
+github.com/sirupsen/logrus v1.5.0
+github.com/spf13/afero v1.2.2
+github.com/stretchr/objx v0.1.0
+github.com/stretchr/testify v1.3.0
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
+golang.org/x/text v0.3.0
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127
+gopkg.in/go-playground/assert.v1 v1.2.1
+gopkg.in/yaml.v2 v2.2.8