Skip to content

Commit

Permalink
feat: support providing parse-as per lockfile / directory
Browse files Browse the repository at this point in the history
  • Loading branch information
@G-Rath
G-Rath committed Apr 27, 2023
1 parent a9867c9 commit af47b6c
Show file tree
Hide file tree
Showing 3 changed files with 220 additions and 16 deletions.
19 changes: 17 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,23 @@ that is not a dependency specification (e.g. flags or files in the case of
`requirements.txt`, though `<properties>` _is_ supported for `pom.xml`)

The detector will attempt to automatically determine the parser to use for each
file based on the filename - you can manually specify the parser to use for all
files with the `-parse-as` flag:
file based on the filename - you can also explicitly specify the parser to use
by prefixing it to the start of the given path, seperated with an `:` symbol:

```shell
osv-detector requirements.txt:path/to/my/requirements/ requirements.txt:path/to/my/file.txt
```

If you have a path with a colon in its name, you can with just a colon to
explicitly signal to the detector that it should infer the parser based on the
filename:

```shell
osv-detector ':/path/to/my:projects/package-lock.json'
```

You can also set the default parser to use for all files with the `--parse-as`
flag:

```shell
osv-detector --parse-as 'package-lock.json' path/to/my/file.lock
Expand Down
47 changes: 34 additions & 13 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"os"
"path/filepath"
"sort"
"strings"
)

// these come from goreleaser
Expand Down Expand Up @@ -283,31 +284,39 @@ func findLockfiles(r *reporter.Reporter, pathToLockOrDirectory string, parseAs s
return lockfiles, err != nil
}

func findAllLockfiles(r *reporter.Reporter, pathsToCheck []string, parseAs string) ([]string, bool) {
func findAllLockfiles(r *reporter.Reporter, pathsToCheck []string, parseAsGlobal string) ([]string, bool) {
var paths []string

if parseAs == parseAsCsvRow {
return []string{"-"}, false
if parseAsGlobal == parseAsCsvRow {
return []string{parseAsCsvRow + ":-"}, false
}

errored := false

for _, pathToLockOrDirectory := range pathsToCheck {
parseAs, pathToLockOrDirectory := parseLockfilePathWithParseAs(pathToLockOrDirectory)

if parseAs == "" {
parseAs = parseAsGlobal
}

lps, erred := findLockfiles(r, pathToLockOrDirectory, parseAs)

if erred {
errored = true
}

for _, p := range lps {
paths = append(paths, filepath.Clean(p))
paths = append(paths, parseAs+":"+filepath.Clean(p))
}
}

return paths, errored
}

func parseLockfile(pathToLock string, parseAs string, args []string) (lockfile.Lockfile, error) {
func parseLockfile(pathToLock string, args []string) (lockfile.Lockfile, error) {
parseAs, pathToLock := parseLockfilePathWithParseAs(pathToLock)

if parseAs == parseAsCsvRow {
l, err := lockfile.FromCSVRows(pathToLock, parseAs, args)

Expand Down Expand Up @@ -405,17 +414,28 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases(
}
}

func parseLockfilePathWithParseAs(lockfilePathWithParseAs string) (string, string) {
if !strings.Contains(lockfilePathWithParseAs, ":") {
lockfilePathWithParseAs = ":" + lockfilePathWithParseAs
}

splits := strings.SplitN(lockfilePathWithParseAs, ":", 2)

return splits[0], splits[1]
}

func readAllLockfiles(
r *reporter.Reporter,
pathsToLocks []string,
parseAs string,
pathsToLocksWithParseAs []string,
args []string,
checkForLocalConfig bool,
config *configer.Config,
) lockfileAndConfigOrErrs {
lockfiles := make([]lockfileAndConfigOrErr, 0, len(pathsToLocks))
lockfiles := make([]lockfileAndConfigOrErr, 0, len(pathsToLocksWithParseAs))

for _, pathToLockWithParseAs := range pathsToLocksWithParseAs {
_, pathToLock := parseLockfilePathWithParseAs(pathToLockWithParseAs)

for _, pathToLock := range pathsToLocks {
if checkForLocalConfig {
base := filepath.Dir(pathToLock)
con, err := configer.Find(r, base)
Expand All @@ -442,7 +462,7 @@ func readAllLockfiles(
}
}

lockf, err := parseLockfile(pathToLock, parseAs, args)
lockf, err := parseLockfile(pathToLockWithParseAs, args)
lockfiles = append(lockfiles, lockfileAndConfigOrErr{lockf, config, err})
}

Expand Down Expand Up @@ -522,6 +542,7 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
return 0
}

// ensure that if the global parseAs is set, it is one of the supported values
if *parseAs != "" && *parseAs != parseAsCsvFile && *parseAs != parseAsCsvRow {
if parser, parsedAs := lockfile.FindParser("", *parseAs); parser == nil {
r.PrintError(fmt.Sprintf("Don't know how to parse files as \"%s\" - supported values are:\n", parsedAs))
Expand All @@ -537,9 +558,9 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
}
}

pathsToLocks, errored := findAllLockfiles(r, cli.Args(), *parseAs)
pathsToLocksWithParseAs, errored := findAllLockfiles(r, cli.Args(), *parseAs)

if len(pathsToLocks) == 0 {
if len(pathsToLocksWithParseAs) == 0 {
r.PrintError(
"You must provide at least one path to either a lockfile or a directory containing at least one lockfile (see --help for usage and flags)\n",
)
Expand Down Expand Up @@ -576,7 +597,7 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
loadLocalConfig = false
}

files := readAllLockfiles(r, pathsToLocks, *parseAs, cli.Args(), loadLocalConfig, &config)
files := readAllLockfiles(r, pathsToLocksWithParseAs, cli.Args(), loadLocalConfig, &config)

files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases)

Expand Down
170 changes: 169 additions & 1 deletion main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,140 @@ func TestRun_DBs(t *testing.T) {
}
}

func TestRun_ParseAs(t *testing.T) {
func TestRun_ParseAsSpecific(t *testing.T) {
t.Parallel()

tests := []cliTestCase{
// when there is just a ":", it defaults as empty
{
name: "",
args: []string{filepath.FromSlash(":./fixtures/locks-insecure/composer.lock")},
wantExitCode: 0,
wantStdout: `
Loaded the following OSV databases:
fixtures/locks-insecure/composer.lock: found 0 packages
no known vulnerabilities found
`,
wantStderr: "",
},
// ":" can be used as an escape (no test though because it's invalid on Windows)
{
name: "",
args: []string{filepath.FromSlash(":./fixtures/locks-insecure/my:file")},
wantExitCode: 127,
wantStdout: "",
wantStderr: `
Error reading ./fixtures/locks-insecure/my:file: open ./fixtures/locks-insecure/my:file: %%
You must provide at least one path to either a lockfile or a directory containing at least one lockfile (see --help for usage and flags)
`,
},
// when a path to a file is given, parse-as is applied to that file
{
name: "",
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-insecure/my-package-lock.json")},
wantExitCode: 1,
wantStdout: `
Loaded the following OSV databases:
npm (%% vulnerabilities, including withdrawn - last updated %%)
fixtures/locks-insecure/my-package-lock.json: found 1 package
Using db npm (%% vulnerabilities, including withdrawn - last updated %%)
ansi-html@0.0.1 is affected by the following vulnerabilities:
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
`,
wantStderr: "",
},
// when a path to a directory is given, parse-as is applied to all files in the directory
{
name: "",
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-insecure")},
wantExitCode: 1,
wantStdout: `
Loaded the following OSV databases:
npm (%% vulnerabilities, including withdrawn - last updated %%)
fixtures/locks-insecure/composer.lock: found 0 packages
no known vulnerabilities found
fixtures/locks-insecure/my-package-lock.json: found 1 package
Using db npm (%% vulnerabilities, including withdrawn - last updated %%)
ansi-html@0.0.1 is affected by the following vulnerabilities:
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
`,
wantStderr: "",
},
// files that error on parsing don't stop parsable files from being checked
{
name: "",
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-empty")},
wantExitCode: 127,
wantStdout: `
Loaded the following OSV databases:
fixtures/locks-empty/composer.lock: found 0 packages
no known vulnerabilities found
`,
wantStderr: `
Error, could not parse fixtures/locks-empty/Gemfile.lock: unexpected end of JSON input
Error, could not parse fixtures/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
`,
},
// files that error on parsing don't stop parsable files from being checked
{
name: "",
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-empty"), filepath.FromSlash("package-lock.json:./fixtures/locks-insecure")},
wantExitCode: 127,
wantStdout: `
Loaded the following OSV databases:
npm (%% vulnerabilities, including withdrawn - last updated %%)
fixtures/locks-empty/composer.lock: found 0 packages
no known vulnerabilities found
fixtures/locks-insecure/composer.lock: found 0 packages
no known vulnerabilities found
fixtures/locks-insecure/my-package-lock.json: found 1 package
Using db npm (%% vulnerabilities, including withdrawn - last updated %%)
ansi-html@0.0.1 is affected by the following vulnerabilities:
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
`,
wantStderr: `
Error, could not parse fixtures/locks-empty/Gemfile.lock: unexpected end of JSON input
Error, could not parse fixtures/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
`,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()

testCli(t, tt)
})
}
}

func TestRun_ParseAsGlobal(t *testing.T) {
t.Parallel()

tests := []cliTestCase{
Expand Down Expand Up @@ -596,6 +729,41 @@ func TestRun_ParseAs(t *testing.T) {
Error, could not parse fixtures/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
`,
},
// specific parse-as takes precedence over global parse-as
{
name: "",
args: []string{"--parse-as", "package-lock.json", filepath.FromSlash("Gemfile.lock:./fixtures/locks-empty"), filepath.FromSlash("./fixtures/locks-insecure")},
wantExitCode: 1,
wantStdout: `
Loaded the following OSV databases:
npm (2971 vulnerabilities, including withdrawn - last updated %%)
fixtures/locks-empty/Gemfile.lock: found 0 packages
no known vulnerabilities found
fixtures/locks-empty/composer.lock: found 0 packages
no known vulnerabilities found
fixtures/locks-empty/yarn.lock: found 0 packages
no known vulnerabilities found
fixtures/locks-insecure/composer.lock: found 0 packages
no known vulnerabilities found
fixtures/locks-insecure/my-package-lock.json: found 1 package
Using db npm (2971 vulnerabilities, including withdrawn - last updated %%)
ansi-html@0.0.1 is affected by the following vulnerabilities:
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
`,
wantStderr: "",
},
}
for _, tt := range tests {
tt := tt
Expand Down

0 comments on commit af47b6c

Please sign in to comment.