Skip to content

Commit

Permalink
gopls/internal/govulncheck: sync x/vuln@62b0186
Browse files Browse the repository at this point in the history
VulnDB OSV schema was changed recently
  https://go-review.googlesource.com/c/vulndb/+/424895
to fix the misinterpretation of 'affected.package.name',
and the database entries were repopulated with the new schema.
We need to update the client library to pick up the change.
We also need to update the fake vulndb entries used in tests.

gopls/regtest/misc/testdata/vulndb was copied from
  golang.org/x/vuln/cmd/govulncheck/testdata/vulndb @ 62b0186
(the version updated in cl/424895)

Also reverse golang.org/cl/425183 which includes the position
information in the SummarizeCallStack result. Like in govulncheck -v,
the position info is already available in the callstack, thus
this is unnecessary for us. Since x/vuln is currently frozen
until the preview release, revert it from gopls/internal/vulncheck.

Ran go mod tidy -compat=1.16; otherwise, the transitive dependency
on github.com/client9/misspell from golang.org/x/vuln breaks go1.16
build.

Updated copy.sh script to copy x/vuln/internal/semver package
(golang/go#54401) and add the build tags back to all go files.
Gopls's builder builds&tests packages with old go versions,
so we still need go1.18 build tag.

Fixes golang/go#54818

Change-Id: I37770d698082378656a7988d3412a4ca2196ca7b
Reviewed-on: https://go-review.googlesource.com/c/tools/+/427542
gopls-CI: kokoro <noreply+kokoro@google.com>
Run-TryBot: Hyang-Ah Hana Kim <hyangah@gmail.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Jonathan Amsterdam <jba@google.com>
  • Loading branch information
hyangah committed Sep 2, 2022
1 parent 655abda commit eb8352e
Show file tree
Hide file tree
Showing 17 changed files with 313 additions and 32 deletions.
4 changes: 3 additions & 1 deletion gopls/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ require (
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4
golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664
golang.org/x/tools v0.1.13-0.20220810174125-0ad49fdeb955
golang.org/x/vuln v0.0.0-20220809164104-12ff722659c1
golang.org/x/vuln v0.0.0-20220901221904-62b0186a1058
honnef.co/go/tools v0.3.3
mvdan.cc/gofumpt v0.3.1
mvdan.cc/xurls/v2 v2.4.0
)

require golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect

require (
github.com/BurntSushi/toml v1.2.0 // indirect
github.com/google/safehtml v0.0.2 // indirect
Expand Down
7 changes: 4 additions & 3 deletions gopls/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnX
github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/google/go-cmdtest v0.4.0/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
Expand Down Expand Up @@ -43,6 +42,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/exp/typeparams v0.0.0-20220722155223-a9213eeb770e h1:7Xs2YCOpMlNqSQSmrrnhlzBXIE/bpMecZplbLePTJvE=
golang.org/x/exp/typeparams v0.0.0-20220722155223-a9213eeb770e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
Expand All @@ -68,8 +69,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/vuln v0.0.0-20220809164104-12ff722659c1 h1:wxIK8Hnmd3ervTxk4aON+gAbfWbb2hToeKSTQd0eXgo=
golang.org/x/vuln v0.0.0-20220809164104-12ff722659c1/go.mod h1:t0tyWMAuNGUOL2N4il/aj/M5LLt4ktPpOGuTBNUqmiM=
golang.org/x/vuln v0.0.0-20220901221904-62b0186a1058 h1:YnB27EXBD8XxB0JcaOeluuvhF2kS4DrH0k+lpopG2xc=
golang.org/x/vuln v0.0.0-20220901221904-62b0186a1058/go.mod h1:7tDfEDtOLlzHQRi4Yzfg5seVBSvouUIjyPzBx4q5CxQ=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
19 changes: 13 additions & 6 deletions gopls/internal/govulncheck/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ package govulncheck
import (
"encoding/json"
"go/build"
"io/ioutil"
"os"
"path/filepath"
"sync"
Expand Down Expand Up @@ -66,7 +65,7 @@ func (c *FSCache) ReadIndex(dbName string) (client.DBIndex, time.Time, error) {
c.mu.Lock()
defer c.mu.Unlock()

b, err := ioutil.ReadFile(filepath.Join(c.rootDir, dbName, "index.json"))
b, err := os.ReadFile(filepath.Join(c.rootDir, dbName, "index.json"))
if err != nil {
if os.IsNotExist(err) {
return nil, time.Time{}, nil
Expand Down Expand Up @@ -95,7 +94,7 @@ func (c *FSCache) WriteIndex(dbName string, index client.DBIndex, retrieved time
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(path, "index.json"), j, 0666); err != nil {
if err := os.WriteFile(filepath.Join(path, "index.json"), j, 0666); err != nil {
return err
}
return nil
Expand All @@ -105,7 +104,11 @@ func (c *FSCache) ReadEntries(dbName string, p string) ([]*osv.Entry, error) {
c.mu.Lock()
defer c.mu.Unlock()

b, err := ioutil.ReadFile(filepath.Join(c.rootDir, dbName, p, "vulns.json"))
ep, err := client.EscapeModulePath(p)
if err != nil {
return nil, err
}
b, err := os.ReadFile(filepath.Join(c.rootDir, dbName, ep, "vulns.json"))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
Expand All @@ -123,15 +126,19 @@ func (c *FSCache) WriteEntries(dbName string, p string, entries []*osv.Entry) er
c.mu.Lock()
defer c.mu.Unlock()

path := filepath.Join(c.rootDir, dbName, p)
ep, err := client.EscapeModulePath(p)
if err != nil {
return err
}
path := filepath.Join(c.rootDir, dbName, ep)
if err := os.MkdirAll(path, 0777); err != nil {
return err
}
j, err := json.Marshal(entries)
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(path, "vulns.json"), j, 0666); err != nil {
if err := os.WriteFile(filepath.Join(path, "vulns.json"), j, 0666); err != nil {
return err
}
return nil
Expand Down
4 changes: 2 additions & 2 deletions gopls/internal/govulncheck/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func TestConcurrency(t *testing.T) {
i := i
g.Go(func() error {
id := i % 5
p := fmt.Sprintf("package%d", id)
p := fmt.Sprintf("example.com/package%d", id)

entries, err := cache.ReadEntries(dbName, p)
if err != nil {
Expand All @@ -115,7 +115,7 @@ func TestConcurrency(t *testing.T) {
// sanity checking
for i := 0; i < 5; i++ {
id := fmt.Sprint(i)
p := fmt.Sprintf("package%s", id)
p := fmt.Sprintf("example.com/package%s", id)

es, err := cache.ReadEntries(dbName, p)
if err != nil {
Expand Down
19 changes: 19 additions & 0 deletions gopls/internal/govulncheck/copy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,22 @@ set -o pipefail

rm -f *.go
cp ../../../../vuln/cmd/govulncheck/internal/govulncheck/*.go .

sed -i '' 's/\"golang.org\/x\/vuln\/internal\/semver\"/\"golang.org\/x\/tools\/gopls\/internal\/govulncheck\/semver\"/g' *.go
sed -i '' -e '4 i\
' -e '4 i\
//go:build go1.18' -e '4 i\
// +build go1.18' *.go

# Copy golang.org/x/vuln/internal/semver that
# golang.org/x/vuln/cmd/govulncheck/internal/govulncheck
# depends on.

mkdir -p semver
cd semver
rm -f *.go
cp ../../../../../vuln/internal/semver/*.go .
sed -i '' -e '4 i\
' -e '4 i\
//go:build go1.18' -e '4 i\
// +build go1.18' *.go
38 changes: 38 additions & 0 deletions gopls/internal/govulncheck/filepath.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build go1.18
// +build go1.18

package govulncheck

import (
"path/filepath"
"strings"
)

// AbsRelShorter takes path and returns its path relative
// to the current directory, if shorter. Returns path
// when path is an empty string or upon any error.
func AbsRelShorter(path string) string {
if path == "" {
return ""
}

c, err := filepath.Abs(".")
if err != nil {
return path
}
r, err := filepath.Rel(c, path)
if err != nil {
return path
}

rSegments := strings.Split(r, string(filepath.Separator))
pathSegments := strings.Split(path, string(filepath.Separator))
if len(rSegments) < len(pathSegments) {
return r
}
return path
}
42 changes: 42 additions & 0 deletions gopls/internal/govulncheck/filepath_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build go1.18
// +build go1.18

package govulncheck

import (
"os"
"path/filepath"
"testing"
)

func TestAbsRelShorter(t *testing.T) {
thisFile := "filepath_test.go"
thisFileAbs, _ := filepath.Abs(thisFile)

tf, err := os.CreateTemp("", "filepath_test.gp")
if err != nil {
t.Errorf("could not create temporary filepath_test.go file: %v", err)
}
tempFile := tf.Name()
tempFileAbs, _ := filepath.Abs(tempFile)

for _, test := range []struct {
l string
want string
}{
{thisFile, "filepath_test.go"},
{thisFileAbs, "filepath_test.go"},
// Relative path to temp file from "." is longer as
// it needs to go back the length of the absolute
// path and then in addition go to os.TempDir.
{tempFile, tempFileAbs},
} {
if got := AbsRelShorter(test.l); got != test.want {
t.Errorf("want %s; got %s", test.want, got)
}
}
}
84 changes: 84 additions & 0 deletions gopls/internal/govulncheck/semver/semver.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build go1.18
// +build go1.18

// Package semver provides shared utilities for manipulating
// Go semantic versions.
package semver

import (
"regexp"
"strings"
)

// addSemverPrefix adds a 'v' prefix to s if it isn't already prefixed
// with 'v' or 'go'. This allows us to easily test go-style SEMVER
// strings against normal SEMVER strings.
func addSemverPrefix(s string) string {
if !strings.HasPrefix(s, "v") && !strings.HasPrefix(s, "go") {
return "v" + s
}
return s
}

// removeSemverPrefix removes the 'v' or 'go' prefixes from go-style
// SEMVER strings, for usage in the public vulnerability format.
func removeSemverPrefix(s string) string {
s = strings.TrimPrefix(s, "v")
s = strings.TrimPrefix(s, "go")
return s
}

// CanonicalizeSemverPrefix turns a SEMVER string into the canonical
// representation using the 'v' prefix, as used by the OSV format.
// Input may be a bare SEMVER ("1.2.3"), Go prefixed SEMVER ("go1.2.3"),
// or already canonical SEMVER ("v1.2.3").
func CanonicalizeSemverPrefix(s string) string {
return addSemverPrefix(removeSemverPrefix(s))
}

var (
// Regexp for matching go tags. The groups are:
// 1 the major.minor version
// 2 the patch version, or empty if none
// 3 the entire prerelease, if present
// 4 the prerelease type ("beta" or "rc")
// 5 the prerelease number
tagRegexp = regexp.MustCompile(`^go(\d+\.\d+)(\.\d+|)((beta|rc|-pre)(\d+))?$`)
)

// This is a modified copy of pkgsite/internal/stdlib:VersionForTag.
func GoTagToSemver(tag string) string {
if tag == "" {
return ""
}

tag = strings.Fields(tag)[0]
// Special cases for go1.
if tag == "go1" {
return "v1.0.0"
}
if tag == "go1.0" {
return ""
}
m := tagRegexp.FindStringSubmatch(tag)
if m == nil {
return ""
}
version := "v" + m[1]
if m[2] != "" {
version += m[2]
} else {
version += ".0"
}
if m[3] != "" {
if !strings.HasPrefix(m[4], "-") {
version += "-"
}
version += m[4] + "." + m[5]
}
return version
}
43 changes: 43 additions & 0 deletions gopls/internal/govulncheck/semver/semver_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build go1.18
// +build go1.18

package semver

import (
"testing"
)

func TestCanonicalize(t *testing.T) {
for _, test := range []struct {
v string
want string
}{
{"v1.2.3", "v1.2.3"},
{"1.2.3", "v1.2.3"},
{"go1.2.3", "v1.2.3"},
} {
got := CanonicalizeSemverPrefix(test.v)
if got != test.want {
t.Errorf("want %s; got %s", test.want, got)
}
}
}

func TestGoTagToSemver(t *testing.T) {
for _, test := range []struct {
v string
want string
}{
{"go1.19", "v1.19.0"},
{"go1.20-pre4", "v1.20.0-pre.4"},
} {
got := GoTagToSemver(test.v)
if got != test.want {
t.Errorf("want %s; got %s", test.want, got)
}
}
}
2 changes: 1 addition & 1 deletion gopls/internal/govulncheck/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func (e *PackageError) Error() string {
var b strings.Builder
fmt.Fprintln(&b, "Packages contain errors:")
for _, e := range e.Errors {
fmt.Println(&b, e)
fmt.Fprintln(&b, e)
}
return b.String()
}
Expand Down

0 comments on commit eb8352e

Please sign in to comment.