From c1d9d1f305df19c66e28a619e17f9f1d7563d977 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 25 May 2018 14:27:29 -0700 Subject: [PATCH 001/187] cmd/go: don't generate output for "go build -o /dev/null x.go" We look for "-o /dev/null", and, if found, pretend that there was no "-o" option and don't generate an action to create the final executable. We look for "go build x.go", and, if found, and if -o was not used, pretend that the user specified "-o x". Unfortunately, we were doing those in the wrong order, so that "go build -o /dev/null x.go" would first clear the "-o" option and then set it to "-o x". This CL flips the order so that the right thing happens. Fixes #25579 Change-Id: Ic9556ac0a57f7b45b685951bc96ba5ea4633b860 Reviewed-on: https://go-review.googlesource.com/114715 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/cmd/go/go_test.go | 11 +++++++++++ src/cmd/go/internal/work/build.go | 10 +++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index fb8846c710dc9..a0fc72aac4af2 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -6325,3 +6325,14 @@ func TestCDAndGOPATHAreDifferent(t *testing.T) { testCDAndGOPATHAreDifferent(tg, cd, strings.ToLower(gopath)) } } + +// Issue 25579. +func TestGoBuildDashODevNull(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.parallel() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata")) + tg.run("build", "-o", os.DevNull, filepath.Join(tg.pwd(), "testdata", "src", "hello", "hello.go")) + tg.mustNotExist("hello") + tg.mustNotExist("hello.exe") +} diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index e5f0e624c3ef9..5cb0c2431f2a8 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -284,11 +284,6 @@ func runBuild(cmd *base.Command, args []string) { cfg.BuildO += cfg.ExeSuffix } - // Special case -o /dev/null by not writing at all. - if cfg.BuildO == os.DevNull { - cfg.BuildO = "" - } - // sanity check some often mis-used options switch cfg.BuildContext.Compiler { case "gccgo": @@ -311,6 +306,11 @@ func runBuild(cmd *base.Command, args []string) { pkgs = pkgsFilter(load.Packages(args)) + // Special case -o /dev/null by not writing at all. + if cfg.BuildO == os.DevNull { + cfg.BuildO = "" + } + if cfg.BuildO != "" { if len(pkgs) > 1 { base.Fatalf("go build: cannot use -o with multiple packages") From 4fe688c6a49e59e852f0bfebbb4cf71366987ce7 Mon Sep 17 00:00:00 2001 From: David Symonds Date: Mon, 28 May 2018 13:03:08 +1000 Subject: [PATCH 002/187] cmd/cover: fix sorting of profile segment boundaries If a span of coverable code is empty (e.g. an empty select clause) then there will be two Boundary values with the same offset. In that case, the starting Boundary needs to come first so that the generated HTML output will open the tag before it tries to close it. Change-Id: Ib44a8b7c36ae57757c18b6cceb7a88ffa4e95394 Reviewed-on: https://go-review.googlesource.com/114855 Reviewed-by: Rob Pike Run-TryBot: Rob Pike TryBot-Result: Gobot Gobot --- src/cmd/cover/profile.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cmd/cover/profile.go b/src/cmd/cover/profile.go index 5628b91f51990..0da42ebfd3570 100644 --- a/src/cmd/cover/profile.go +++ b/src/cmd/cover/profile.go @@ -174,7 +174,7 @@ func (p *Profile) Boundaries(src []byte) (boundaries []Boundary) { return b } if max <= 1 { - b.Norm = 0.8 // Profile is in"set" mode; we want a heat map. Use cov8 in the CSS. + b.Norm = 0.8 // Profile is in "set" mode; we want a heat map. Use cov8 in the CSS. } else if count > 0 { b.Norm = math.Log(float64(count)) / divisor } @@ -209,7 +209,10 @@ func (b boundariesByPos) Len() int { return len(b) } func (b boundariesByPos) Swap(i, j int) { b[i], b[j] = b[j], b[i] } func (b boundariesByPos) Less(i, j int) bool { if b[i].Offset == b[j].Offset { - return !b[i].Start && b[j].Start + // Boundaries at the same offset should be ordered Start < !Start. + // They represent empty sections of code (e.g. a switch/select clause + // without a body). + return b[i].Start && !b[j].Start } return b[i].Offset < b[j].Offset } From 68de5508d374c910cc0bf066b8f65cd5395115b1 Mon Sep 17 00:00:00 2001 From: Yury Smolsky Date: Tue, 22 May 2018 22:37:40 +0300 Subject: [PATCH 003/187] cmd/vet: eliminate use of Perl in tests This change uses errorCheck and wantedErrors functions copied from the test/run.go to eliminate use of the test/errchk perl script. Tests' error messages that contained full filenames were changed to have base filenames because the errorCheck function processes output from "go vet" in the same way. Fixes #20032. Change-Id: Ieb7be67c2d7281b9648171c698398449b7e2d4dd Reviewed-on: https://go-review.googlesource.com/114176 Run-TryBot: Yury Smolsky TryBot-Result: Gobot Gobot Reviewed-by: Alan Donovan --- src/cmd/vet/testdata/atomic.go | 2 +- src/cmd/vet/testdata/shadow.go | 8 +- src/cmd/vet/testdata/structtag.go | 18 +- src/cmd/vet/vet_test.go | 285 +++++++++++++++++++++++++----- 4 files changed, 255 insertions(+), 58 deletions(-) diff --git a/src/cmd/vet/testdata/atomic.go b/src/cmd/vet/testdata/atomic.go index 8b587567c75e0..69730b4e6f03f 100644 --- a/src/cmd/vet/testdata/atomic.go +++ b/src/cmd/vet/testdata/atomic.go @@ -43,7 +43,7 @@ func AtomicTests() { { // A variable declaration creates a new variable in the current scope. - x := atomic.AddUint64(&x, 1) // ERROR "declaration of .x. shadows declaration at testdata/atomic.go:16" + x := atomic.AddUint64(&x, 1) // ERROR "declaration of .x. shadows declaration at atomic.go:16" // Re-declaration assigns a new value. x, w := atomic.AddUint64(&x, 1), 10 // ERROR "direct assignment to atomic value" diff --git a/src/cmd/vet/testdata/shadow.go b/src/cmd/vet/testdata/shadow.go index 3b61137b87c09..c55cb2772a9c6 100644 --- a/src/cmd/vet/testdata/shadow.go +++ b/src/cmd/vet/testdata/shadow.go @@ -17,7 +17,7 @@ func ShadowRead(f *os.File, buf []byte) (err error) { _ = err } if f != nil { - _, err := f.Read(buf) // ERROR "declaration of .err. shadows declaration at testdata/shadow.go:13" + _, err := f.Read(buf) // ERROR "declaration of .err. shadows declaration at shadow.go:13" if err != nil { return err } @@ -25,8 +25,8 @@ func ShadowRead(f *os.File, buf []byte) (err error) { _ = i } if f != nil { - x := one() // ERROR "declaration of .x. shadows declaration at testdata/shadow.go:14" - var _, err = f.Read(buf) // ERROR "declaration of .err. shadows declaration at testdata/shadow.go:13" + x := one() // ERROR "declaration of .x. shadows declaration at shadow.go:14" + var _, err = f.Read(buf) // ERROR "declaration of .err. shadows declaration at shadow.go:13" if x == 1 && err != nil { return err } @@ -46,7 +46,7 @@ func ShadowRead(f *os.File, buf []byte) (err error) { if shadowTemp := shadowTemp; true { // OK: obviously intentional idiomatic redeclaration var f *os.File // OK because f is not mentioned later in the function. // The declaration of x is a shadow because x is mentioned below. - var x int // ERROR "declaration of .x. shadows declaration at testdata/shadow.go:14" + var x int // ERROR "declaration of .x. shadows declaration at shadow.go:14" _, _, _ = x, f, shadowTemp } // Use a couple of variables to trigger shadowing errors. diff --git a/src/cmd/vet/testdata/structtag.go b/src/cmd/vet/testdata/structtag.go index c87e42f5d004b..ce21e803c802e 100644 --- a/src/cmd/vet/testdata/structtag.go +++ b/src/cmd/vet/testdata/structtag.go @@ -44,40 +44,40 @@ type AnonymousXML struct{} type DuplicateJSONFields struct { JSON int `json:"a"` - DuplicateJSON int `json:"a"` // ERROR "struct field DuplicateJSON repeats json tag .a. also at testdata/structtag.go:46" + DuplicateJSON int `json:"a"` // ERROR "struct field DuplicateJSON repeats json tag .a. also at structtag.go:46" IgnoredJSON int `json:"-"` OtherIgnoredJSON int `json:"-"` OmitJSON int `json:",omitempty"` OtherOmitJSON int `json:",omitempty"` - DuplicateOmitJSON int `json:"a,omitempty"` // ERROR "struct field DuplicateOmitJSON repeats json tag .a. also at testdata/structtag.go:46" + DuplicateOmitJSON int `json:"a,omitempty"` // ERROR "struct field DuplicateOmitJSON repeats json tag .a. also at structtag.go:46" NonJSON int `foo:"a"` DuplicateNonJSON int `foo:"a"` Embedded struct { DuplicateJSON int `json:"a"` // OK because its not in the same struct type } - AnonymousJSON `json:"a"` // ERROR "struct field AnonymousJSON repeats json tag .a. also at testdata/structtag.go:46" + AnonymousJSON `json:"a"` // ERROR "struct field AnonymousJSON repeats json tag .a. also at structtag.go:46" XML int `xml:"a"` - DuplicateXML int `xml:"a"` // ERROR "struct field DuplicateXML repeats xml tag .a. also at testdata/structtag.go:60" + DuplicateXML int `xml:"a"` // ERROR "struct field DuplicateXML repeats xml tag .a. also at structtag.go:60" IgnoredXML int `xml:"-"` OtherIgnoredXML int `xml:"-"` OmitXML int `xml:",omitempty"` OtherOmitXML int `xml:",omitempty"` - DuplicateOmitXML int `xml:"a,omitempty"` // ERROR "struct field DuplicateOmitXML repeats xml tag .a. also at testdata/structtag.go:60" + DuplicateOmitXML int `xml:"a,omitempty"` // ERROR "struct field DuplicateOmitXML repeats xml tag .a. also at structtag.go:60" NonXML int `foo:"a"` DuplicateNonXML int `foo:"a"` Embedded struct { DuplicateXML int `xml:"a"` // OK because its not in the same struct type } - AnonymousXML `xml:"a"` // ERROR "struct field AnonymousXML repeats xml tag .a. also at testdata/structtag.go:60" + AnonymousXML `xml:"a"` // ERROR "struct field AnonymousXML repeats xml tag .a. also at structtag.go:60" Attribute struct { XMLName xml.Name `xml:"b"` NoDup int `xml:"b"` // OK because XMLName above affects enclosing struct. Attr int `xml:"b,attr"` // OK because 0 is valid. - DupAttr int `xml:"b,attr"` // ERROR "struct field DupAttr repeats xml attribute tag .b. also at testdata/structtag.go:76" - DupOmitAttr int `xml:"b,omitempty,attr"` // ERROR "struct field DupOmitAttr repeats xml attribute tag .b. also at testdata/structtag.go:76" + DupAttr int `xml:"b,attr"` // ERROR "struct field DupAttr repeats xml attribute tag .b. also at structtag.go:76" + DupOmitAttr int `xml:"b,omitempty,attr"` // ERROR "struct field DupOmitAttr repeats xml attribute tag .b. also at structtag.go:76" - AnonymousXML `xml:"b,attr"` // ERROR "struct field AnonymousXML repeats xml attribute tag .b. also at testdata/structtag.go:76" + AnonymousXML `xml:"b,attr"` // ERROR "struct field AnonymousXML repeats xml attribute tag .b. also at structtag.go:76" } } diff --git a/src/cmd/vet/vet_test.go b/src/cmd/vet/vet_test.go index f654d4679ec12..ecb4ce129566e 100644 --- a/src/cmd/vet/vet_test.go +++ b/src/cmd/vet/vet_test.go @@ -6,12 +6,17 @@ package main_test import ( "bytes" + "errors" "fmt" "internal/testenv" + "io/ioutil" + "log" "os" "os/exec" "path/filepath" + "regexp" "runtime" + "strconv" "strings" "sync" "testing" @@ -19,7 +24,7 @@ import ( const ( dataDir = "testdata" - binary = "testvet.exe" + binary = "./testvet.exe" ) // We implement TestMain so remove the test binary when all is done. @@ -29,16 +34,6 @@ func TestMain(m *testing.M) { os.Exit(result) } -func MustHavePerl(t *testing.T) { - switch runtime.GOOS { - case "plan9", "windows": - t.Skipf("skipping test: perl not available on %s", runtime.GOOS) - } - if _, err := exec.LookPath("perl"); err != nil { - t.Skipf("skipping test: perl not found in path") - } -} - var ( buildMu sync.Mutex // guards following built = false // We have built the binary. @@ -55,7 +50,6 @@ func Build(t *testing.T) { t.Skip("cannot run on this environment") } testenv.MustHaveGoBuild(t) - MustHavePerl(t) cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", binary) output, err := cmd.CombinedOutput() if err != nil { @@ -67,23 +61,19 @@ func Build(t *testing.T) { } func Vet(t *testing.T, files []string) { - errchk := filepath.Join(runtime.GOROOT(), "test", "errchk") flags := []string{ - "./" + binary, "-printfuncs=Warn:1,Warnf:1", "-all", "-shadow", } - cmd := exec.Command(errchk, append(flags, files...)...) - if !run(cmd, t) { - t.Fatal("vet command failed") - } + cmd := exec.Command(binary, append(flags, files...)...) + errchk(cmd, files, t) } -// Run this shell script, but do it in Go so it can be run by "go test". -// go build -o testvet -// $(GOROOT)/test/errchk ./testvet -shadow -printfuncs='Warn:1,Warnf:1' testdata/*.go testdata/*.s -// rm testvet +// TestVet is equivalent to running this: +// go build -o ./testvet +// errorCheck the output of ./testvet -shadow -printfuncs='Warn:1,Warnf:1' testdata/*.go testdata/*.s +// rm ./testvet // // TestVet tests self-contained files in testdata/*.go. @@ -95,7 +85,6 @@ func TestVet(t *testing.T) { Build(t) t.Parallel() - // errchk ./testvet gos, err := filepath.Glob(filepath.Join(dataDir, "*.go")) if err != nil { t.Fatal(err) @@ -128,17 +117,14 @@ func TestVet(t *testing.T) { func TestVetPrint(t *testing.T) { Build(t) - errchk := filepath.Join(runtime.GOROOT(), "test", "errchk") + file := filepath.Join("testdata", "print.go") cmd := exec.Command( - errchk, - "go", "vet", "-vettool=./"+binary, + "go", "vet", "-vettool="+binary, "-printf", "-printfuncs=Warn:1,Warnf:1", - "testdata/print.go", + file, ) - if !run(cmd, t) { - t.Fatal("vet command failed") - } + errchk(cmd, []string{file}, t) } func TestVetAsm(t *testing.T) { @@ -155,7 +141,6 @@ func TestVetAsm(t *testing.T) { } t.Parallel() - // errchk ./testvet Vet(t, append(gos, asms...)) } @@ -181,23 +166,20 @@ func TestVetDirs(t *testing.T) { } } -func run(c *exec.Cmd, t *testing.T) bool { +func errchk(c *exec.Cmd, files []string, t *testing.T) { output, err := c.CombinedOutput() - if err != nil { + if _, ok := err.(*exec.ExitError); !ok { t.Logf("vet output:\n%s", output) t.Fatal(err) } - // Errchk delights by not returning non-zero status if it finds errors, so we look at the output. - // It prints "BUG" if there is a failure. - if !c.ProcessState.Success() { - t.Logf("vet output:\n%s", output) - return false + fullshort := make([]string, 0, len(files)*2) + for _, f := range files { + fullshort = append(fullshort, f, filepath.Base(f)) } - ok := !bytes.Contains(output, []byte("BUG")) - if !ok { - t.Logf("vet output:\n%s", output) + err = errorCheck(string(output), false, fullshort...) + if err != nil { + t.Errorf("error check failed: %s", err) } - return ok } // TestTags verifies that the -tags argument controls which files to check. @@ -214,7 +196,7 @@ func TestTags(t *testing.T) { "-v", // We're going to look at the files it examines. "testdata/tagtest", } - cmd := exec.Command("./"+binary, args...) + cmd := exec.Command(binary, args...) output, err := cmd.CombinedOutput() if err != nil { t.Fatal(err) @@ -234,10 +216,225 @@ func TestTags(t *testing.T) { func TestVetVerbose(t *testing.T) { t.Parallel() Build(t) - cmd := exec.Command("./"+binary, "-v", "-all", "testdata/cgo/cgo3.go") + cmd := exec.Command(binary, "-v", "-all", "testdata/cgo/cgo3.go") out, err := cmd.CombinedOutput() if err != nil { t.Logf("%s", out) t.Error(err) } } + +// All declarations below were adapted from test/run.go. + +// errorCheck matches errors in outStr against comments in source files. +// For each line of the source files which should generate an error, +// there should be a comment of the form // ERROR "regexp". +// If outStr has an error for a line which has no such comment, +// this function will report an error. +// Likewise if outStr does not have an error for a line which has a comment, +// or if the error message does not match the . +// The syntax is Perl but its best to stick to egrep. +// +// Sources files are supplied as fullshort slice. +// It consists of pairs: full path to source file and it's base name. +func errorCheck(outStr string, wantAuto bool, fullshort ...string) (err error) { + var errs []error + out := splitOutput(outStr, wantAuto) + // Cut directory name. + for i := range out { + for j := 0; j < len(fullshort); j += 2 { + full, short := fullshort[j], fullshort[j+1] + out[i] = strings.Replace(out[i], full, short, -1) + } + } + + var want []wantedError + for j := 0; j < len(fullshort); j += 2 { + full, short := fullshort[j], fullshort[j+1] + want = append(want, wantedErrors(full, short)...) + } + for _, we := range want { + var errmsgs []string + if we.auto { + errmsgs, out = partitionStrings("", out) + } else { + errmsgs, out = partitionStrings(we.prefix, out) + } + if len(errmsgs) == 0 { + errs = append(errs, fmt.Errorf("%s:%d: missing error %q", we.file, we.lineNum, we.reStr)) + continue + } + matched := false + n := len(out) + for _, errmsg := range errmsgs { + // Assume errmsg says "file:line: foo". + // Cut leading "file:line: " to avoid accidental matching of file name instead of message. + text := errmsg + if i := strings.Index(text, " "); i >= 0 { + text = text[i+1:] + } + if we.re.MatchString(text) { + matched = true + } else { + out = append(out, errmsg) + } + } + if !matched { + errs = append(errs, fmt.Errorf("%s:%d: no match for %#q in:\n\t%s", we.file, we.lineNum, we.reStr, strings.Join(out[n:], "\n\t"))) + continue + } + } + + if len(out) > 0 { + errs = append(errs, fmt.Errorf("Unmatched Errors:")) + for _, errLine := range out { + errs = append(errs, fmt.Errorf("%s", errLine)) + } + } + + if len(errs) == 0 { + return nil + } + if len(errs) == 1 { + return errs[0] + } + var buf bytes.Buffer + fmt.Fprintf(&buf, "\n") + for _, err := range errs { + fmt.Fprintf(&buf, "%s\n", err.Error()) + } + return errors.New(buf.String()) +} + +func splitOutput(out string, wantAuto bool) []string { + // gc error messages continue onto additional lines with leading tabs. + // Split the output at the beginning of each line that doesn't begin with a tab. + // lines are impossible to match so those are filtered out. + var res []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSuffix(line, "\r") // normalize Windows output + if strings.HasPrefix(line, "\t") { + res[len(res)-1] += "\n" + line + } else if strings.HasPrefix(line, "go tool") || strings.HasPrefix(line, "#") || !wantAuto && strings.HasPrefix(line, "") { + continue + } else if strings.TrimSpace(line) != "" { + res = append(res, line) + } + } + return res +} + +// matchPrefix reports whether s starts with file name prefix followed by a :, +// and possibly preceded by a directory name. +func matchPrefix(s, prefix string) bool { + i := strings.Index(s, ":") + if i < 0 { + return false + } + j := strings.LastIndex(s[:i], "/") + s = s[j+1:] + if len(s) <= len(prefix) || s[:len(prefix)] != prefix { + return false + } + if s[len(prefix)] == ':' { + return true + } + return false +} + +func partitionStrings(prefix string, strs []string) (matched, unmatched []string) { + for _, s := range strs { + if matchPrefix(s, prefix) { + matched = append(matched, s) + } else { + unmatched = append(unmatched, s) + } + } + return +} + +type wantedError struct { + reStr string + re *regexp.Regexp + lineNum int + auto bool // match line + file string + prefix string +} + +var ( + errRx = regexp.MustCompile(`// (?:GC_)?ERROR (.*)`) + errAutoRx = regexp.MustCompile(`// (?:GC_)?ERRORAUTO (.*)`) + errQuotesRx = regexp.MustCompile(`"([^"]*)"`) + lineRx = regexp.MustCompile(`LINE(([+-])([0-9]+))?`) +) + +// wantedErrors parses expected errors from comments in a file. +func wantedErrors(file, short string) (errs []wantedError) { + cache := make(map[string]*regexp.Regexp) + + src, err := ioutil.ReadFile(file) + if err != nil { + log.Fatal(err) + } + for i, line := range strings.Split(string(src), "\n") { + lineNum := i + 1 + if strings.Contains(line, "////") { + // double comment disables ERROR + continue + } + var auto bool + m := errAutoRx.FindStringSubmatch(line) + if m != nil { + auto = true + } else { + m = errRx.FindStringSubmatch(line) + } + if m == nil { + continue + } + all := m[1] + mm := errQuotesRx.FindAllStringSubmatch(all, -1) + if mm == nil { + log.Fatalf("%s:%d: invalid errchk line: %s", file, lineNum, line) + } + for _, m := range mm { + replacedOnce := false + rx := lineRx.ReplaceAllStringFunc(m[1], func(m string) string { + if replacedOnce { + return m + } + replacedOnce = true + n := lineNum + if strings.HasPrefix(m, "LINE+") { + delta, _ := strconv.Atoi(m[5:]) + n += delta + } else if strings.HasPrefix(m, "LINE-") { + delta, _ := strconv.Atoi(m[5:]) + n -= delta + } + return fmt.Sprintf("%s:%d", short, n) + }) + re := cache[rx] + if re == nil { + var err error + re, err = regexp.Compile(rx) + if err != nil { + log.Fatalf("%s:%d: invalid regexp \"%#q\" in ERROR line: %v", file, lineNum, rx, err) + } + cache[rx] = re + } + prefix := fmt.Sprintf("%s:%d", short, lineNum) + errs = append(errs, wantedError{ + reStr: rx, + re: re, + prefix: prefix, + auto: auto, + lineNum: lineNum, + file: short, + }) + } + } + + return +} From 1170d946060cb6b6267c14ed87abbe30c7b8309a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=20R=C3=B8dseth?= Date: Mon, 28 May 2018 10:40:54 +0000 Subject: [PATCH 004/187] crypto: fix typos in the comments * Fix typos in the comments in the assembly code for the crypto package. Change-Id: Iac146a7d8bee4a680a8d4d3af533fbc1b259482d GitHub-Last-Rev: 65090a38956df4c14bf55df4881c76e8c3d32447 GitHub-Pull-Request: golang/go#25606 Reviewed-on: https://go-review.googlesource.com/114803 Reviewed-by: Brad Fitzpatrick --- src/crypto/elliptic/p256_asm_s390x.s | 2 +- src/crypto/sha1/sha1block_amd64.s | 4 ++-- src/crypto/sha256/sha256block_amd64.s | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crypto/elliptic/p256_asm_s390x.s b/src/crypto/elliptic/p256_asm_s390x.s index d0e6d09e2e326..8a17e81062fc5 100644 --- a/src/crypto/elliptic/p256_asm_s390x.s +++ b/src/crypto/elliptic/p256_asm_s390x.s @@ -991,7 +991,7 @@ TEXT ·p256OrdMul(SB), NOSPLIT, $0 * *Mi obra de arte de siglo XXI @vpaprots * * - * First group is special, doesnt get the two inputs: + * First group is special, doesn't get the two inputs: * +--------+--------+<-+ * +-------| ADD2 | ADD1 |--|-----+ * | +--------+--------+ | | diff --git a/src/crypto/sha1/sha1block_amd64.s b/src/crypto/sha1/sha1block_amd64.s index a0032c4544095..135f113898c43 100644 --- a/src/crypto/sha1/sha1block_amd64.s +++ b/src/crypto/sha1/sha1block_amd64.s @@ -395,7 +395,7 @@ end: PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x60,0x240) \ PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x60,0x260) -// Macros calculating individual rounds have general forn +// Macros calculating individual rounds have general form // CALC_ROUND_PRE + PRECALC_ROUND + CALC_ROUND_POST // CALC_ROUND_{PRE,POST} macros follow @@ -413,7 +413,7 @@ end: LEAL (REG_E)(R12*1), REG_E // E += A >>> 5 -// Registers are cycleickly rotated DX -> AX -> DI -> SI -> BX -> CX +// Registers are cyclically rotated DX -> AX -> DI -> SI -> BX -> CX #define CALC_0 \ MOVL SI, BX \ // Precalculating first round RORXL $2, SI, SI \ diff --git a/src/crypto/sha256/sha256block_amd64.s b/src/crypto/sha256/sha256block_amd64.s index f30f4829a6c1b..f533f64260781 100644 --- a/src/crypto/sha256/sha256block_amd64.s +++ b/src/crypto/sha256/sha256block_amd64.s @@ -755,7 +755,7 @@ avx2_loop1: // for w0 - w47 JB avx2_loop1 avx2_loop2: - // w48 - w63 processed with no scheduliung (last 16 rounds) + // w48 - w63 processed with no scheduling (last 16 rounds) VPADDD 0*32(TBL)(SRND*1), XDWORD0, XFER VMOVDQU XFER, (_XFER + 0*32)(SP)(SRND*1) DO_ROUND_N_0(_XFER + 0*32, a, b, c, d, e, f, g, h, h) From 6070e432d2cbde0f04b7391069f5dbb27589b341 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Mon, 28 May 2018 11:36:29 -0700 Subject: [PATCH 005/187] cmd/dist: report correct CC name in error about missing CC Fixes #25611 Change-Id: I487463584e4d9d99cf5d8db0c9a4b4e66464ecd8 Reviewed-on: https://go-review.googlesource.com/114935 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/cmd/dist/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index 99d1db5909590..effea903e9de7 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -1438,7 +1438,7 @@ func checkCC() { fatalf("cannot invoke C compiler %q: %v\n\n"+ "Go needs a system C compiler for use with cgo.\n"+ "To set a C compiler, set CC=the-compiler.\n"+ - "To disable cgo, set CGO_ENABLED=0.\n%s%s", defaultcc, err, outputHdr, output) + "To disable cgo, set CGO_ENABLED=0.\n%s%s", defaultcc[""], err, outputHdr, output) } } From 002c764533bd15414ce26b99570815bd5e48b16e Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Mon, 28 May 2018 15:29:01 -0700 Subject: [PATCH 006/187] test: gofmt bounds.go Change-Id: I8b462e20064658120afc8eb1cbac926254d1e24e Reviewed-on: https://go-review.googlesource.com/114937 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- test/bounds.go | 92 +++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/test/bounds.go b/test/bounds.go index a0febb515ac4b..34c444877baae 100644 --- a/test/bounds.go +++ b/test/bounds.go @@ -12,23 +12,23 @@ package foo var ( s []int - a1 [1]int - a1k [1000]int + a1 [1]int + a1k [1000]int a100k [100000]int - p1 *[1]int - p1k *[1000]int + p1 *[1]int + p1k *[1000]int p100k *[100000]int - i int - ui uint - i8 int8 - ui8 uint8 - i16 int16 + i int + ui uint + i8 int8 + ui8 uint8 + i16 int16 ui16 uint16 - i32 int32 + i32 int32 ui32 uint32 - i64 int64 + i64 int64 ui64 uint64 ) @@ -61,11 +61,11 @@ func main() { // Unsigned 8-bit numbers don't need checks for len >= 2⁸. use(s[ui8]) use(a1[ui8]) - use(a1k[ui8]) // ERROR "index bounds check elided" - use(a100k[ui8]) // ERROR "index bounds check elided" + use(a1k[ui8]) // ERROR "index bounds check elided" + use(a100k[ui8]) // ERROR "index bounds check elided" use(p1[ui8]) - use(p1k[ui8]) // ERROR "index bounds check elided" - use(p100k[ui8]) // ERROR "index bounds check elided" + use(p1k[ui8]) // ERROR "index bounds check elided" + use(p100k[ui8]) // ERROR "index bounds check elided" use(s[i16]) use(a1[i16]) @@ -79,10 +79,10 @@ func main() { use(s[ui16]) use(a1[ui16]) use(a1k[ui16]) - use(a100k[ui16]) // ERROR "index bounds check elided" + use(a100k[ui16]) // ERROR "index bounds check elided" use(p1[ui16]) use(p1k[ui16]) - use(p100k[ui16]) // ERROR "index bounds check elided" + use(p100k[ui16]) // ERROR "index bounds check elided" use(s[i32]) use(a1[i32]) @@ -128,11 +128,11 @@ func main() { use(s[ui%999]) use(a1[ui%999]) - use(a1k[ui%999]) // ERROR "index bounds check elided" - use(a100k[ui%999]) // ERROR "index bounds check elided" + use(a1k[ui%999]) // ERROR "index bounds check elided" + use(a100k[ui%999]) // ERROR "index bounds check elided" use(p1[ui%999]) - use(p1k[ui%999]) // ERROR "index bounds check elided" - use(p100k[ui%999]) // ERROR "index bounds check elided" + use(p1k[ui%999]) // ERROR "index bounds check elided" + use(p100k[ui%999]) // ERROR "index bounds check elided" use(s[i%1000]) use(a1[i%1000]) @@ -144,11 +144,11 @@ func main() { use(s[ui%1000]) use(a1[ui%1000]) - use(a1k[ui%1000]) // ERROR "index bounds check elided" - use(a100k[ui%1000]) // ERROR "index bounds check elided" + use(a1k[ui%1000]) // ERROR "index bounds check elided" + use(a100k[ui%1000]) // ERROR "index bounds check elided" use(p1[ui%1000]) - use(p1k[ui%1000]) // ERROR "index bounds check elided" - use(p100k[ui%1000]) // ERROR "index bounds check elided" + use(p1k[ui%1000]) // ERROR "index bounds check elided" + use(p100k[ui%1000]) // ERROR "index bounds check elided" use(s[i%1001]) use(a1[i%1001]) @@ -161,45 +161,45 @@ func main() { use(s[ui%1001]) use(a1[ui%1001]) use(a1k[ui%1001]) - use(a100k[ui%1001]) // ERROR "index bounds check elided" + use(a100k[ui%1001]) // ERROR "index bounds check elided" use(p1[ui%1001]) use(p1k[ui%1001]) - use(p100k[ui%1001]) // ERROR "index bounds check elided" + use(p100k[ui%1001]) // ERROR "index bounds check elided" // Bitwise and truncates the maximum value to the mask value. // The result (for a positive mask) cannot be negative, so elision // applies to both signed and unsigned indexes. use(s[i&999]) use(a1[i&999]) - use(a1k[i&999]) // ERROR "index bounds check elided" - use(a100k[i&999]) // ERROR "index bounds check elided" + use(a1k[i&999]) // ERROR "index bounds check elided" + use(a100k[i&999]) // ERROR "index bounds check elided" use(p1[i&999]) - use(p1k[i&999]) // ERROR "index bounds check elided" - use(p100k[i&999]) // ERROR "index bounds check elided" + use(p1k[i&999]) // ERROR "index bounds check elided" + use(p100k[i&999]) // ERROR "index bounds check elided" use(s[ui&999]) use(a1[ui&999]) - use(a1k[ui&999]) // ERROR "index bounds check elided" - use(a100k[ui&999]) // ERROR "index bounds check elided" + use(a1k[ui&999]) // ERROR "index bounds check elided" + use(a100k[ui&999]) // ERROR "index bounds check elided" use(p1[ui&999]) - use(p1k[ui&999]) // ERROR "index bounds check elided" - use(p100k[ui&999]) // ERROR "index bounds check elided" + use(p1k[ui&999]) // ERROR "index bounds check elided" + use(p100k[ui&999]) // ERROR "index bounds check elided" use(s[i&1000]) use(a1[i&1000]) use(a1k[i&1000]) - use(a100k[i&1000]) // ERROR "index bounds check elided" + use(a100k[i&1000]) // ERROR "index bounds check elided" use(p1[i&1000]) use(p1k[i&1000]) - use(p100k[i&1000]) // ERROR "index bounds check elided" + use(p100k[i&1000]) // ERROR "index bounds check elided" use(s[ui&1000]) use(a1[ui&1000]) use(a1k[ui&1000]) - use(a100k[ui&1000]) // ERROR "index bounds check elided" + use(a100k[ui&1000]) // ERROR "index bounds check elided" use(p1[ui&1000]) use(p1k[ui&1000]) - use(p100k[ui&1000]) // ERROR "index bounds check elided" + use(p100k[ui&1000]) // ERROR "index bounds check elided" // Right shift cuts the effective number of bits in the index, // but only for unsigned (signed stays negative). @@ -214,10 +214,10 @@ func main() { use(s[ui32>>22]) use(a1[ui32>>22]) use(a1k[ui32>>22]) - use(a100k[ui32>>22]) // ERROR "index bounds check elided" + use(a100k[ui32>>22]) // ERROR "index bounds check elided" use(p1[ui32>>22]) use(p1k[ui32>>22]) - use(p100k[ui32>>22]) // ERROR "index bounds check elided" + use(p100k[ui32>>22]) // ERROR "index bounds check elided" use(s[i32>>23]) use(a1[i32>>23]) @@ -229,11 +229,11 @@ func main() { use(s[ui32>>23]) use(a1[ui32>>23]) - use(a1k[ui32>>23]) // ERROR "index bounds check elided" - use(a100k[ui32>>23]) // ERROR "index bounds check elided" + use(a1k[ui32>>23]) // ERROR "index bounds check elided" + use(a100k[ui32>>23]) // ERROR "index bounds check elided" use(p1[ui32>>23]) - use(p1k[ui32>>23]) // ERROR "index bounds check elided" - use(p100k[ui32>>23]) // ERROR "index bounds check elided" + use(p1k[ui32>>23]) // ERROR "index bounds check elided" + use(p100k[ui32>>23]) // ERROR "index bounds check elided" // Division cuts the range like right shift does. use(s[i/1e6]) @@ -263,7 +263,7 @@ func main() { use(p1[ui/1e7]) } -var sum int +var sum int func use(x int) { sum += x From 33cfcf6afac867f4fbf94237b9219778e5fd51c7 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Mon, 28 May 2018 13:49:40 -0700 Subject: [PATCH 007/187] cmd/compile: fix trivial typos in comments Change-Id: I04880d87e317a1140ec12da6ec5e788991719760 Reviewed-on: https://go-review.googlesource.com/114936 Reviewed-by: Ian Lance Taylor --- src/cmd/compile/internal/ssa/branchelim.go | 2 +- src/cmd/compile/internal/ssa/gen/generic.rules | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd/compile/internal/ssa/branchelim.go b/src/cmd/compile/internal/ssa/branchelim.go index 1967fb93febfd..d9dcaf844430d 100644 --- a/src/cmd/compile/internal/ssa/branchelim.go +++ b/src/cmd/compile/internal/ssa/branchelim.go @@ -4,7 +4,7 @@ package ssa -// branchelim tries to elminiate branches by +// branchelim tries to eliminate branches by // generating CondSelect instructions. // // Search for basic blocks that look like diff --git a/src/cmd/compile/internal/ssa/gen/generic.rules b/src/cmd/compile/internal/ssa/gen/generic.rules index 76212c0866853..7931aa7f06d2f 100644 --- a/src/cmd/compile/internal/ssa/gen/generic.rules +++ b/src/cmd/compile/internal/ssa/gen/generic.rules @@ -212,7 +212,7 @@ // (Mod64u x y) is always between 0 (inclusive) and y (exclusive). (IsInBounds (Mod32u _ y) y) -> (ConstBool [1]) (IsInBounds (Mod64u _ y) y) -> (ConstBool [1]) -// Right shifting a unsigned number limits its value. +// Right shifting an unsigned number limits its value. (IsInBounds (ZeroExt8to64 (Rsh8Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 8 && 1< (ConstBool [1]) (IsInBounds (ZeroExt8to32 (Rsh8Ux64 _ (Const64 [c]))) (Const32 [d])) && 0 < c && c < 8 && 1< (ConstBool [1]) (IsInBounds (ZeroExt8to16 (Rsh8Ux64 _ (Const64 [c]))) (Const16 [d])) && 0 < c && c < 8 && 1< (ConstBool [1]) From 5c36fdfd39e231e87b267fc718c9fd6a86f49b90 Mon Sep 17 00:00:00 2001 From: Aarti Parikh Date: Tue, 29 May 2018 00:42:10 +0000 Subject: [PATCH 008/187] image: add an example that shows how to get the metadata of an image This is a simple but everyday use case in image libraries. Currently, there is one example in this library and it is lengthy and involved. This PR will be imported into Gerrit with the title and first comment (this text) used to generate the subject and body of the Gerrit change. Change-Id: Idca527d97c095af88755446e1548fa2b8ace7eb0 GitHub-Last-Rev: f5743c8ef337a8b7a5126db21d784593f4f53c8a GitHub-Pull-Request: golang/go#25616 Reviewed-on: https://go-review.googlesource.com/114939 Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/image/decode_example_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/image/decode_example_test.go b/src/image/decode_example_test.go index 81fa0378e171e..526c03f3c1a78 100644 --- a/src/image/decode_example_test.go +++ b/src/image/decode_example_test.go @@ -21,6 +21,15 @@ import ( _ "image/jpeg" ) +func Example_decodeConfig() { + reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(data)) + config, format, err := image.DecodeConfig(reader) + if err != nil { + log.Fatal(err) + } + fmt.Println("Width:", config.Width, "Height:", config.Height, "Format:", format) +} + func Example() { // Decode the JPEG data. If reading from file, create a reader with // From 53209467dbb94a37ac5550945a59f5d0721976d8 Mon Sep 17 00:00:00 2001 From: Zhou Peng Date: Tue, 29 May 2018 08:56:42 +0000 Subject: [PATCH 009/187] index/suffixarray: fix a typo mistake in comments Change-Id: Ibdd1ca7bfc6fb2419621338f1f8e37c876ba89c0 Reviewed-on: https://go-review.googlesource.com/114976 Reviewed-by: Alberto Donizetti --- src/index/suffixarray/suffixarray.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/suffixarray/suffixarray.go b/src/index/suffixarray/suffixarray.go index c59ae6eef1fd2..0961ac4fb21be 100644 --- a/src/index/suffixarray/suffixarray.go +++ b/src/index/suffixarray/suffixarray.go @@ -107,7 +107,7 @@ func (x *Index) Read(r io.Reader) error { // allocate space if 2*n < cap(x.data) || cap(x.data) < n { - // new data is significantly smaller or larger then + // new data is significantly smaller or larger than // existing buffers - allocate new ones x.data = make([]byte, n) x.sa = make([]int, n) From 99695bd90b70ae562349a15b3a0b00d78c51e23f Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Tue, 29 May 2018 09:08:42 +0200 Subject: [PATCH 010/187] runtime: fix typo in comment for cpuinit Change-Id: Ie19e0354f813b042c995c0221072568eaa473bea Reviewed-on: https://go-review.googlesource.com/114995 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/proc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index e312c575d0fb4..8a5812639ebb5 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -479,7 +479,7 @@ func internal_cpu_initialize(env string) //go:linkname internal_cpu_debugOptions internal/cpu.debugOptions var internal_cpu_debugOptions bool -// cpuinit extracts the environment variable GODEBUGCPU from the enviroment on +// cpuinit extracts the environment variable GODEBUGCPU from the environment on // Linux and Darwin if the GOEXPERIMENT debugcpu was set and calls internal/cpu.initialize. func cpuinit() { const prefix = "GODEBUGCPU=" From ab922e1012f75bdb7cff041b0da4bb16a40c64c2 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Tue, 29 May 2018 06:40:56 -0700 Subject: [PATCH 011/187] cmd/cgo: use standard generated code comment Change cgo to follow https://golang.org/s/generatedcode. For the C code we continue to use /* */ comments, so they don't follow the format exactly. It doesn't really matter since the format is only for Go code anyhow. This CL changes the C code to be similar for consistency. Fixes #25623 Change-Id: Idcfee53ec4069924d173ab8cedeb7bcfb7312863 Reviewed-on: https://go-review.googlesource.com/115035 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick Reviewed-by: Tobias Klauser --- src/cmd/cgo/godefs.go | 2 +- src/cmd/cgo/out.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cmd/cgo/godefs.go b/src/cmd/cgo/godefs.go index 6d638f064422a..6720945cdddba 100644 --- a/src/cmd/cgo/godefs.go +++ b/src/cmd/cgo/godefs.go @@ -19,7 +19,7 @@ import ( func (p *Package) godefs(f *File, srcfile string) string { var buf bytes.Buffer - fmt.Fprintf(&buf, "// Created by cgo -godefs - DO NOT EDIT\n") + fmt.Fprintf(&buf, "// Code generated by cmd/cgo -godefs; DO NOT EDIT.\n") fmt.Fprintf(&buf, "// %s %s\n", filepath.Base(os.Args[0]), strings.Join(os.Args[1:], " ")) fmt.Fprintf(&buf, "\n") diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 399e96d01e0e7..62ef872ca0da8 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -73,7 +73,7 @@ func (p *Package) writeDefs() { // Write second Go output: definitions of _C_xxx. // In a separate file so that the import of "unsafe" does not // pollute the original file. - fmt.Fprintf(fgo2, "// Created by cgo - DO NOT EDIT\n\n") + fmt.Fprintf(fgo2, "// Code generated by cmd/cgo; DO NOT EDIT.\n\n") fmt.Fprintf(fgo2, "package %s\n\n", p.PackageName) fmt.Fprintf(fgo2, "import \"unsafe\"\n\n") if !*gccgo && *importRuntimeCgo { @@ -536,7 +536,7 @@ func (p *Package) writeOutput(f *File, srcfile string) { p.GccFiles = append(p.GccFiles, base+".cgo2.c") // Write Go output: Go input with rewrites of C.xxx to _C_xxx. - fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n\n") + fmt.Fprintf(fgo1, "// Code generated by cmd/cgo; DO NOT EDIT.\n\n") fmt.Fprintf(fgo1, "//line %s:1\n", srcfile) fgo1.Write(f.Edit.Bytes()) @@ -740,7 +740,7 @@ func (p *Package) packedAttribute() string { func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { p.writeExportHeader(fgcch) - fmt.Fprintf(fgcc, "/* Created by cgo - DO NOT EDIT. */\n") + fmt.Fprintf(fgcc, "/* Code generated by cmd/cgo; DO NOT EDIT. */\n\n") fmt.Fprintf(fgcc, "#include \n") fmt.Fprintf(fgcc, "#include \"_cgo_export.h\"\n\n") @@ -981,7 +981,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { p.writeExportHeader(fgcch) - fmt.Fprintf(fgcc, "/* Created by cgo - DO NOT EDIT. */\n") + fmt.Fprintf(fgcc, "/* Code generated by cmd/cgo; DO NOT EDIT. */\n\n") fmt.Fprintf(fgcc, "#include \"_cgo_export.h\"\n") fmt.Fprintf(fgcc, "%s\n", gccgoExportFileProlog) @@ -1147,7 +1147,7 @@ func (p *Package) writeGccgoExports(fgo2, fm, fgcc, fgcch io.Writer) { // writeExportHeader writes out the start of the _cgo_export.h file. func (p *Package) writeExportHeader(fgcch io.Writer) { - fmt.Fprintf(fgcch, "/* Created by \"go tool cgo\" - DO NOT EDIT. */\n\n") + fmt.Fprintf(fgcch, "/* Code generated by cmd/cgo; DO NOT EDIT. */\n\n") pkg := *importPath if pkg == "" { pkg = p.PackagePath From d15d0550544fc6392ff58a99939eeb907a823737 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 29 May 2018 09:30:12 -0700 Subject: [PATCH 012/187] cmd/compile: reject large argument areas Extend stack frame limit of 1GB to include large argument/return areas. Argument/return areas are part of the parent frame, not the frame itself, so they need to be handled separately. Fixes #25507. Change-Id: I309298a58faee3e7c1dac80bd2f1166c82460087 Reviewed-on: https://go-review.googlesource.com/115036 Run-TryBot: Keith Randall Reviewed-by: Josh Bleecher Snyder TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/pgen.go | 3 ++- src/go/types/stdlib_test.go | 1 + test/fixedbugs/issue25507.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue25507.go diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go index 9747a0299e672..8f3947b0a686a 100644 --- a/src/cmd/compile/internal/gc/pgen.go +++ b/src/cmd/compile/internal/gc/pgen.go @@ -257,7 +257,8 @@ const maxStackSize = 1 << 30 // worker indicates which of the backend workers is doing the processing. func compileSSA(fn *Node, worker int) { f := buildssa(fn, worker) - if f.Frontend().(*ssafn).stksize >= maxStackSize { + // Note: check arg size to fix issue 25507. + if f.Frontend().(*ssafn).stksize >= maxStackSize || fn.Type.ArgWidth() >= maxStackSize { largeStackFramesMu.Lock() largeStackFrames = append(largeStackFrames, fn.Pos) largeStackFramesMu.Unlock() diff --git a/src/go/types/stdlib_test.go b/src/go/types/stdlib_test.go index ad4c51f74def1..dd1510d37e369 100644 --- a/src/go/types/stdlib_test.go +++ b/src/go/types/stdlib_test.go @@ -174,6 +174,7 @@ func TestStdFixed(t *testing.T) { "issue20529.go", // go/types does not have constraints on stack size "issue22200.go", // go/types does not have constraints on stack size "issue22200b.go", // go/types does not have constraints on stack size + "issue25507.go", // go/types does not have constraints on stack size ) } diff --git a/test/fixedbugs/issue25507.go b/test/fixedbugs/issue25507.go new file mode 100644 index 0000000000000..8dcbae16ab9ee --- /dev/null +++ b/test/fixedbugs/issue25507.go @@ -0,0 +1,29 @@ +// errorcheck + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// We have a limit of 1GB for stack frames. +// Test that we extend that limit to include large argument/return areas. +// Argument/return areas are part of the parent frame, not the frame itself, +// so they need to be handled separately. + +package main + +// >1GB to trigger failure, <2GB to work on 32-bit platforms. +type large struct { + b [1500000000]byte +} + +func (x large) f1() int { // ERROR "stack frame too large" + return 5 +} + +func f2(x large) int { // ERROR "stack frame too large" + return 5 +} + +func f3() (x large, i int) { // ERROR "stack frame too large" + return +} From 07d384b9de6b58bfe8cf9c6543179654353944b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Fri, 4 May 2018 11:22:53 +0700 Subject: [PATCH 013/187] cmd/vet: avoid false positives with non-comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vet's buildtag check looks for malformed build tag comments. Since these can appear in Go files as well as non-Go files (such as assembly files), it must read the file line by line instead of using go/token or go/ast directly. However, this method runs into false positives if there are any lines in the code that look like comments, but are not. For example: $ cat f.go package main const foo = ` //+build ignore ` $ go vet f.go ./f.go:3: +build comment must appear before package clause and be followed by a blank line This bug has been popping up more frequently since vet started being run with go test, so it is important to make the check as precise as possible. To avoid the false positive, when checking a Go file, cross-check that a line that looks like a comment actually corresponds to a comment in the go/ast syntax tree. Since vet already obtains the syntax trees for all the Go files, it checks, this change means very little extra work for the check. While at it, add a badf helper function to simplify the code that reports warnings in the buildtag check. Fixes #13533. Change-Id: I484a16da01363b409ec418c313634171bf85250b Reviewed-on: https://go-review.googlesource.com/111415 Run-TryBot: Daniel Martí TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/vet/buildtag.go | 62 ++++++++++++++++++----- src/cmd/vet/main.go | 9 ++-- src/cmd/vet/testdata/buildtag/buildtag.go | 4 ++ 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/cmd/vet/buildtag.go b/src/cmd/vet/buildtag.go index 80d8f819240c4..d1fedec554918 100644 --- a/src/cmd/vet/buildtag.go +++ b/src/cmd/vet/buildtag.go @@ -19,11 +19,39 @@ var ( ) // checkBuildTag checks that build tags are in the correct location and well-formed. -func checkBuildTag(name string, data []byte) { +func checkBuildTag(f *File) { if !vet("buildtags") { return } - lines := bytes.SplitAfter(data, nl) + // badf is like File.Badf, but it uses a line number instead of + // token.Pos. + badf := func(line int, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args) + fmt.Fprintf(os.Stderr, "%s:%d: %s\n", f.name, line, msg) + setExit(1) + } + + // we must look at the raw lines, as build tags may appear in non-Go + // files such as assembly files. + lines := bytes.SplitAfter(f.content, nl) + + // lineWithComment reports whether a line corresponds to a comment in + // the source file. If the source file wasn't Go, the function always + // returns true. + lineWithComment := func(line int) bool { + if f.file == nil { + // Current source file is not Go, so be conservative. + return true + } + for _, group := range f.file.Comments { + startLine := f.fset.Position(group.Pos()).Line + endLine := f.fset.Position(group.End()).Line + if startLine <= line && line <= endLine { + return true + } + } + return false + } // Determine cutpoint where +build comments are no longer valid. // They are valid in leading // comments in the file followed by @@ -46,18 +74,29 @@ func checkBuildTag(name string, data []byte) { if !bytes.HasPrefix(line, slashSlash) { continue } + if !bytes.Contains(line, plusBuild) { + // Check that the comment contains "+build" early, to + // avoid unnecessary lineWithComment calls that may + // incur linear searches. + continue + } + if !lineWithComment(i + 1) { + // This is a line in a Go source file that looks like a + // comment, but actually isn't - such as part of a raw + // string. + continue + } + text := bytes.TrimSpace(line[2:]) if bytes.HasPrefix(text, plusBuild) { fields := bytes.Fields(text) if !bytes.Equal(fields[0], plusBuild) { // Comment is something like +buildasdf not +build. - fmt.Fprintf(os.Stderr, "%s:%d: possible malformed +build comment\n", name, i+1) - setExit(1) + badf(i+1, "possible malformed +build comment") continue } if i >= cutoff { - fmt.Fprintf(os.Stderr, "%s:%d: +build comment must appear before package clause and be followed by a blank line\n", name, i+1) - setExit(1) + badf(i+1, "+build comment must appear before package clause and be followed by a blank line") continue } // Check arguments. @@ -65,15 +104,13 @@ func checkBuildTag(name string, data []byte) { for _, arg := range fields[1:] { for _, elem := range strings.Split(string(arg), ",") { if strings.HasPrefix(elem, "!!") { - fmt.Fprintf(os.Stderr, "%s:%d: invalid double negative in build constraint: %s\n", name, i+1, arg) - setExit(1) + badf(i+1, "invalid double negative in build constraint: %s", arg) break Args } elem = strings.TrimPrefix(elem, "!") for _, c := range elem { if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' && c != '.' { - fmt.Fprintf(os.Stderr, "%s:%d: invalid non-alphanumeric build constraint: %s\n", name, i+1, arg) - setExit(1) + badf(i+1, "invalid non-alphanumeric build constraint: %s", arg) break Args } } @@ -82,9 +119,8 @@ func checkBuildTag(name string, data []byte) { continue } // Comment with +build but not at beginning. - if bytes.Contains(line, plusBuild) && i < cutoff { - fmt.Fprintf(os.Stderr, "%s:%d: possible malformed +build comment\n", name, i+1) - setExit(1) + if i < cutoff { + badf(i+1, "possible malformed +build comment") continue } } diff --git a/src/cmd/vet/main.go b/src/cmd/vet/main.go index 4422add72f242..50af846c59980 100644 --- a/src/cmd/vet/main.go +++ b/src/cmd/vet/main.go @@ -415,23 +415,24 @@ func doPackage(names []string, basePkg *Package) *Package { warnf("%s: %s", name, err) return nil } - checkBuildTag(name, data) var parsedFile *ast.File if strings.HasSuffix(name, ".go") { - parsedFile, err = parser.ParseFile(fs, name, data, 0) + parsedFile, err = parser.ParseFile(fs, name, data, parser.ParseComments) if err != nil { warnf("%s: %s", name, err) return nil } astFiles = append(astFiles, parsedFile) } - files = append(files, &File{ + file := &File{ fset: fs, content: data, name: name, file: parsedFile, dead: make(map[ast.Node]bool), - }) + } + checkBuildTag(file) + files = append(files, file) } if len(astFiles) == 0 { return nil diff --git a/src/cmd/vet/testdata/buildtag/buildtag.go b/src/cmd/vet/testdata/buildtag/buildtag.go index f12f895dfb19b..6ee08da638da8 100644 --- a/src/cmd/vet/testdata/buildtag/buildtag.go +++ b/src/cmd/vet/testdata/buildtag/buildtag.go @@ -12,3 +12,7 @@ package testdata // +build toolate // ERROR "build comment must appear before package clause and be followed by a blank line" var _ = 3 + +var _ = ` +// +build notacomment +` From 377567edd01572a3ce7ff263fd7923a5ba7a9217 Mon Sep 17 00:00:00 2001 From: Alberto Donizetti Date: Tue, 29 May 2018 14:36:28 +0200 Subject: [PATCH 014/187] reflect: document that StructOf panics on unexported fields Fixes #25401 Change-Id: I0b61ecfcee43ebfe0a84b5c1e28a3817f96b94ed Reviewed-on: https://go-review.googlesource.com/115015 Reviewed-by: Brad Fitzpatrick --- src/reflect/type.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/reflect/type.go b/src/reflect/type.go index a78b20d8f49e3..1f3b665ce4da9 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -2345,8 +2345,9 @@ func isValidFieldName(fieldName string) bool { // The Offset and Index fields are ignored and computed as they would be // by the compiler. // -// StructOf currently does not generate wrapper methods for embedded fields. -// This limitation may be lifted in a future version. +// StructOf currently does not generate wrapper methods for embedded +// fields and panics if passed unexported StructFields. +// These limitations may be lifted in a future version. func StructOf(fields []StructField) Type { var ( hash = fnv1(0, []byte("struct {")...) From b65934f237f80242a907c46ef9cf103ad6959e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 29 May 2018 17:10:39 +0200 Subject: [PATCH 015/187] cmd/go: whitelist -v as a compiler and linker flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's harmless, and can be useful to see what's happening under the hood. Fixes #24593. Change-Id: Iacff378471e86c33aa048161cd65c504709fb339 Reviewed-on: https://go-review.googlesource.com/115075 Run-TryBot: Daniel Martí TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/go/internal/work/security.go | 2 ++ src/cmd/go/internal/work/security_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go index 5c67aa945eff7..880f4fdc791a3 100644 --- a/src/cmd/go/internal/work/security.go +++ b/src/cmd/go/internal/work/security.go @@ -94,6 +94,7 @@ var validCompilerFlags = []*regexp.Regexp{ re(`-?-stdlib=([^@\-].*)`), re(`-w`), re(`-x([^@\-].*)`), + re(`-v`), } var validCompilerFlagsWithNextArg = []string{ @@ -127,6 +128,7 @@ var validLinkerFlags = []*regexp.Regexp{ re(`-shared`), re(`-?-static([-a-z0-9+]*)`), re(`-?-stdlib=([^@\-].*)`), + re(`-v`), // Note that any wildcards in -Wl need to exclude comma, // since -Wl splits its argument at commas and passes diff --git a/src/cmd/go/internal/work/security_test.go b/src/cmd/go/internal/work/security_test.go index bd898c9de6c17..15eeff9b4b4bc 100644 --- a/src/cmd/go/internal/work/security_test.go +++ b/src/cmd/go/internal/work/security_test.go @@ -57,6 +57,7 @@ var goodCompilerFlags = [][]string{ {"-I", "世界"}, {"-framework", "Chocolate"}, {"-x", "c"}, + {"-v"}, } var badCompilerFlags = [][]string{ @@ -132,6 +133,7 @@ var goodLinkerFlags = [][]string{ {"-l", "世界"}, {"-L", "framework"}, {"-framework", "Chocolate"}, + {"-v"}, {"-Wl,-framework", "-Wl,Chocolate"}, {"-Wl,-framework,Chocolate"}, {"-Wl,-unresolved-symbols=ignore-all"}, From db9341a02424a80a93ad50b362b881544b685e19 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 29 May 2018 10:30:54 -0700 Subject: [PATCH 016/187] cmd/compile: update WBLoads during deadcode When we deadcode-remove a block which is a write barrier test, remove that block from the list of write barrier test blocks. Fixes #25516 Change-Id: I1efe732d5476003eab4ad6bf67d0340d7874ff0c Reviewed-on: https://go-review.googlesource.com/115037 Run-TryBot: Keith Randall Reviewed-by: Austin Clements TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/ssa/deadcode.go | 13 ++++++++++++ test/fixedbugs/issue25516.go | 26 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 test/fixedbugs/issue25516.go diff --git a/src/cmd/compile/internal/ssa/deadcode.go b/src/cmd/compile/internal/ssa/deadcode.go index 322ea82c8d603..13b7d7e1e84a6 100644 --- a/src/cmd/compile/internal/ssa/deadcode.go +++ b/src/cmd/compile/internal/ssa/deadcode.go @@ -252,6 +252,19 @@ func deadcode(f *Func) { b.Values = b.Values[:i] } + // Remove dead blocks from WBLoads list. + i = 0 + for _, b := range f.WBLoads { + if reachable[b.ID] { + f.WBLoads[i] = b + i++ + } + } + for j := i; j < len(f.WBLoads); j++ { + f.WBLoads[j] = nil + } + f.WBLoads = f.WBLoads[:i] + // Remove unreachable blocks. Return dead blocks to allocator. i = 0 for _, b := range f.Blocks { diff --git a/test/fixedbugs/issue25516.go b/test/fixedbugs/issue25516.go new file mode 100644 index 0000000000000..8326ef953f08c --- /dev/null +++ b/test/fixedbugs/issue25516.go @@ -0,0 +1,26 @@ +// compile + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure dead write barriers are handled correctly. + +package main + +func f(p **int) { + // The trick here is to eliminate the block containing the write barrier, + // but only after the write barrier branches are inserted. + // This requires some delicate code. + i := 0 + var b []bool + var s string + for true { + if b[i] { + var a []string + s = a[len(s)] + i = 0 + } + *p = nil + } +} From ef53de8ba4d774e4eedb04192a9f03f28a9652e8 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 26 May 2018 11:29:55 -0700 Subject: [PATCH 017/187] runtime: wrap darwin libc calls to keep profiler happy The profiler reports "ExternalCode" when a profiler interrupt happens while in libc code. Instead, keep track of the most recent Go frame for the profiler to use. There is a test for this using time.Now (runtime.TestTimePprof), which will work once time.Now is moved to using libc (my next CL). Change-Id: I940ea83edada482a482e2ab103d3a65589979464 Reviewed-on: https://go-review.googlesource.com/114798 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/sys_darwin.go | 58 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 57790c140cb98..124b2fee5e22e 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -6,48 +6,74 @@ package runtime import "unsafe" +// Call fn with arg as its argument. Return what fn returns. +// fn is the raw pc value of the entry point of the desired function. +// Switches to the system stack, if not already there. +// Preserves the calling point as the location where a profiler traceback will begin. +//go:nosplit +func libcCall(fn, arg unsafe.Pointer) int32 { + // Leave caller's PC/SP/G around for traceback. + gp := getg() + var mp *m + if gp != nil { + mp = gp.m + } + if mp != nil { + mp.libcallg.set(gp) + mp.libcallpc = getcallerpc() + // sp must be the last, because once async cpu profiler finds + // all three values to be non-zero, it will use them + mp.libcallsp = getcallersp() + } + res := asmcgocall(fn, arg) + if mp != nil { + mp.libcallsp = 0 + } + return res +} + // The *_trampoline functions convert from the Go calling convention to the C calling convention // and then call the underlying libc function. They are defined in sys_darwin_$ARCH.s. //go:nosplit //go:cgo_unsafe_args func pthread_attr_init(attr *pthreadattr) int32 { - return asmcgocall(unsafe.Pointer(funcPC(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) + return libcCall(unsafe.Pointer(funcPC(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) } func pthread_attr_init_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_attr_setstacksize(attr *pthreadattr, size uintptr) int32 { - return asmcgocall(unsafe.Pointer(funcPC(pthread_attr_setstacksize_trampoline)), unsafe.Pointer(&attr)) + return libcCall(unsafe.Pointer(funcPC(pthread_attr_setstacksize_trampoline)), unsafe.Pointer(&attr)) } func pthread_attr_setstacksize_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_attr_setdetachstate(attr *pthreadattr, state int) int32 { - return asmcgocall(unsafe.Pointer(funcPC(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) + return libcCall(unsafe.Pointer(funcPC(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) } func pthread_attr_setdetachstate_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32 { - return asmcgocall(unsafe.Pointer(funcPC(pthread_create_trampoline)), unsafe.Pointer(&attr)) + return libcCall(unsafe.Pointer(funcPC(pthread_create_trampoline)), unsafe.Pointer(&attr)) } func pthread_create_trampoline() //go:nosplit //go:cgo_unsafe_args func raise(sig uint32) { - asmcgocall(unsafe.Pointer(funcPC(raise_trampoline)), unsafe.Pointer(&sig)) + libcCall(unsafe.Pointer(funcPC(raise_trampoline)), unsafe.Pointer(&sig)) } func raise_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_self() (t pthread) { - asmcgocall(unsafe.Pointer(funcPC(pthread_self_trampoline)), unsafe.Pointer(&t)) + libcCall(unsafe.Pointer(funcPC(pthread_self_trampoline)), unsafe.Pointer(&t)) return } func pthread_self_trampoline() @@ -61,7 +87,7 @@ func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (un ret1 unsafe.Pointer ret2 int }{addr, n, prot, flags, fd, off, nil, 0} - asmcgocall(unsafe.Pointer(funcPC(mmap_trampoline)), unsafe.Pointer(&args)) + libcCall(unsafe.Pointer(funcPC(mmap_trampoline)), unsafe.Pointer(&args)) return args.ret1, args.ret2 } func mmap_trampoline() @@ -69,56 +95,56 @@ func mmap_trampoline() //go:nosplit //go:cgo_unsafe_args func munmap(addr unsafe.Pointer, n uintptr) { - asmcgocall(unsafe.Pointer(funcPC(munmap_trampoline)), unsafe.Pointer(&addr)) + libcCall(unsafe.Pointer(funcPC(munmap_trampoline)), unsafe.Pointer(&addr)) } func munmap_trampoline() //go:nosplit //go:cgo_unsafe_args func madvise(addr unsafe.Pointer, n uintptr, flags int32) { - asmcgocall(unsafe.Pointer(funcPC(madvise_trampoline)), unsafe.Pointer(&addr)) + libcCall(unsafe.Pointer(funcPC(madvise_trampoline)), unsafe.Pointer(&addr)) } func madvise_trampoline() //go:nosplit //go:cgo_unsafe_args func read(fd int32, p unsafe.Pointer, n int32) int32 { - return asmcgocall(unsafe.Pointer(funcPC(read_trampoline)), unsafe.Pointer(&fd)) + return libcCall(unsafe.Pointer(funcPC(read_trampoline)), unsafe.Pointer(&fd)) } func read_trampoline() //go:nosplit //go:cgo_unsafe_args func closefd(fd int32) int32 { - return asmcgocall(unsafe.Pointer(funcPC(close_trampoline)), unsafe.Pointer(&fd)) + return libcCall(unsafe.Pointer(funcPC(close_trampoline)), unsafe.Pointer(&fd)) } func close_trampoline() //go:nosplit //go:cgo_unsafe_args func exit(code int32) { - asmcgocall(unsafe.Pointer(funcPC(exit_trampoline)), unsafe.Pointer(&code)) + libcCall(unsafe.Pointer(funcPC(exit_trampoline)), unsafe.Pointer(&code)) } func exit_trampoline() //go:nosplit //go:cgo_unsafe_args func usleep(usec uint32) { - asmcgocall(unsafe.Pointer(funcPC(usleep_trampoline)), unsafe.Pointer(&usec)) + libcCall(unsafe.Pointer(funcPC(usleep_trampoline)), unsafe.Pointer(&usec)) } func usleep_trampoline() //go:nosplit //go:cgo_unsafe_args func write(fd uintptr, p unsafe.Pointer, n int32) int32 { - return asmcgocall(unsafe.Pointer(funcPC(write_trampoline)), unsafe.Pointer(&fd)) + return libcCall(unsafe.Pointer(funcPC(write_trampoline)), unsafe.Pointer(&fd)) } func write_trampoline() //go:nosplit //go:cgo_unsafe_args func open(name *byte, mode, perm int32) (ret int32) { - return asmcgocall(unsafe.Pointer(funcPC(open_trampoline)), unsafe.Pointer(&name)) + return libcCall(unsafe.Pointer(funcPC(open_trampoline)), unsafe.Pointer(&name)) } func open_trampoline() @@ -129,7 +155,7 @@ func nanotime() int64 { t int64 // raw timer numer, denom uint32 // conversion factors. nanoseconds = t * numer / denom. } - asmcgocall(unsafe.Pointer(funcPC(nanotime_trampoline)), unsafe.Pointer(&r)) + libcCall(unsafe.Pointer(funcPC(nanotime_trampoline)), unsafe.Pointer(&r)) // Note: Apple seems unconcerned about overflow here. See // https://developer.apple.com/library/content/qa/qa1398/_index.html // Note also, numer == denom == 1 is common. From 23b687eccbcc68f44acf34fd5971f0f5c04d90de Mon Sep 17 00:00:00 2001 From: Dave Russell Date: Sat, 19 May 2018 00:54:43 +0000 Subject: [PATCH 018/187] bytes: re-slice buffer to its previous length after call to grow() Fixes #25435 The added test fails without the re-slice and passes with it. Change-Id: I5ebc2a737285eb116ecc5938d8bf49050652830f GitHub-Last-Rev: 454ddad7df8d56a1d0e05a999ed8277c5516ce01 GitHub-Pull-Request: golang/go#25436 Reviewed-on: https://go-review.googlesource.com/113495 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/bytes/buffer.go | 1 + src/bytes/buffer_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/bytes/buffer.go b/src/bytes/buffer.go index dc9d5e95d32ba..a2eca2ed129bb 100644 --- a/src/bytes/buffer.go +++ b/src/bytes/buffer.go @@ -202,6 +202,7 @@ func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) { b.lastRead = opInvalid for { i := b.grow(MinRead) + b.buf = b.buf[:i] m, e := r.Read(b.buf[i:cap(b.buf)]) if m < 0 { panic(errNegativeRead) diff --git a/src/bytes/buffer_test.go b/src/bytes/buffer_test.go index e4bbc12f6a135..acbe5ca0c49e2 100644 --- a/src/bytes/buffer_test.go +++ b/src/bytes/buffer_test.go @@ -269,6 +269,39 @@ func TestReadFrom(t *testing.T) { } } +type panicReader struct{ panic bool } + +func (r panicReader) Read(p []byte) (int, error) { + if r.panic { + panic(nil) + } + return 0, io.EOF +} + +// Make sure that an empty Buffer remains empty when +// it is "grown" before a Read that panics +func TestReadFromPanicReader(t *testing.T) { + + // First verify non-panic behaviour + var buf Buffer + i, err := buf.ReadFrom(panicReader{}) + if err != nil { + t.Fatal(err) + } + if i != 0 { + t.Fatalf("unexpected return from bytes.ReadFrom (1): got: %d, want %d", i, 0) + } + check(t, "TestReadFromPanicReader (1)", &buf, "") + + // Confirm that when Reader panics, the emtpy buffer remains empty + var buf2 Buffer + defer func() { + recover() + check(t, "TestReadFromPanicReader (2)", &buf2, "") + }() + buf2.ReadFrom(panicReader{panic: true}) +} + func TestReadFromNegativeReader(t *testing.T) { var b Buffer defer func() { From 479018150c0575d46495689315448f5524813ed5 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 29 May 2018 18:57:44 +0000 Subject: [PATCH 019/187] all: remove support for macOS 10.9 and earlier Updates #23122 Change-Id: I4c12ec5cb1a1f15d7858f3deab636710c0660e26 Reviewed-on: https://go-review.googlesource.com/115038 Run-TryBot: Brad Fitzpatrick Reviewed-by: Ian Lance Taylor --- src/crypto/x509/root_cgo_darwin.go | 57 ------------------------------ src/net/fd_unix.go | 13 ------- src/runtime/crash_cgo_test.go | 13 ------- 3 files changed, 83 deletions(-) diff --git a/src/crypto/x509/root_cgo_darwin.go b/src/crypto/x509/root_cgo_darwin.go index 5c310bff09122..11515b82d00e9 100644 --- a/src/crypto/x509/root_cgo_darwin.go +++ b/src/crypto/x509/root_cgo_darwin.go @@ -16,59 +16,6 @@ package x509 #include #include -// FetchPEMRoots_MountainLion is the version of FetchPEMRoots from Go 1.6 -// which still works on OS X 10.8 (Mountain Lion). -// It lacks support for admin & user cert domains. -// See golang.org/issue/16473 -int FetchPEMRoots_MountainLion(CFDataRef *pemRoots) { - if (pemRoots == NULL) { - return -1; - } - CFArrayRef certs = NULL; - OSStatus err = SecTrustCopyAnchorCertificates(&certs); - if (err != noErr) { - return -1; - } - CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0); - int i, ncerts = CFArrayGetCount(certs); - for (i = 0; i < ncerts; i++) { - CFDataRef data = NULL; - SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, i); - if (cert == NULL) { - continue; - } - // Note: SecKeychainItemExport is deprecated as of 10.7 in favor of SecItemExport. - // Once we support weak imports via cgo we should prefer that, and fall back to this - // for older systems. - err = SecKeychainItemExport(cert, kSecFormatX509Cert, kSecItemPemArmour, NULL, &data); - if (err != noErr) { - continue; - } - if (data != NULL) { - CFDataAppendBytes(combinedData, CFDataGetBytePtr(data), CFDataGetLength(data)); - CFRelease(data); - } - } - CFRelease(certs); - *pemRoots = combinedData; - return 0; -} - -// useOldCode reports whether the running machine is OS X 10.8 Mountain Lion -// or older. We only support Mountain Lion and higher, but we'll at least try our -// best on older machines and continue to use the old code path. -// -// See golang.org/issue/16473 -int useOldCode() { - char str[256]; - size_t size = sizeof(str); - memset(str, 0, size); - sysctlbyname("kern.osrelease", str, &size, NULL, 0); - // OS X 10.8 is osrelease "12.*", 10.7 is 11.*, 10.6 is 10.*. - // We never supported things before that. - return memcmp(str, "12.", 3) == 0 || memcmp(str, "11.", 3) == 0 || memcmp(str, "10.", 3) == 0; -} - // FetchPEMRoots fetches the system's list of trusted X.509 root certificates. // // On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root @@ -80,10 +27,6 @@ int useOldCode() { int FetchPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots) { int i; - if (useOldCode()) { - return FetchPEMRoots_MountainLion(pemRoots); - } - // Get certificates from all domains, not just System, this lets // the user add CAs to their "login" keychain, and Admins to add // to the "System" keychain diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index 145933e7377ef..3dcbeb51c7402 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -263,19 +263,6 @@ var tryDupCloexec = int32(1) func dupCloseOnExec(fd int) (newfd int, err error) { if atomic.LoadInt32(&tryDupCloexec) == 1 { r0, _, e1 := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_DUPFD_CLOEXEC, 0) - if runtime.GOOS == "darwin" && e1 == syscall.EBADF { - // On OS X 10.6 and below (but we only support - // >= 10.6), F_DUPFD_CLOEXEC is unsupported - // and fcntl there falls back (undocumented) - // to doing an ioctl instead, returning EBADF - // in this case because fd is not of the - // expected device fd type. Treat it as - // EINVAL instead, so we fall back to the - // normal dup path. - // TODO: only do this on 10.6 if we can detect 10.6 - // cheaply. - e1 = syscall.EINVAL - } switch e1 { case 0: return int(r0), nil diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index fc0cfd9aed18e..d8f75a468b047 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -89,19 +89,6 @@ func TestCgoExternalThreadSIGPROF(t *testing.T) { switch runtime.GOOS { case "plan9", "windows": t.Skipf("no pthreads on %s", runtime.GOOS) - case "darwin": - if runtime.GOARCH != "arm" && runtime.GOARCH != "arm64" { - // static constructor needs external linking, but we don't support - // external linking on OS X 10.6. - out, err := exec.Command("uname", "-r").Output() - if err != nil { - t.Fatalf("uname -r failed: %v", err) - } - // OS X 10.6 == Darwin 10.x - if strings.HasPrefix(string(out), "10.") { - t.Skipf("no external linking on OS X 10.6") - } - } } if runtime.GOARCH == "ppc64" { // TODO(austin) External linking not implemented on From 39888635acd8fb7d54d1bec41262cf5861a5ebfd Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 29 May 2018 19:45:34 +0000 Subject: [PATCH 020/187] net/http: document how Hijack and Request.Context interact Fixes #22347 Change-Id: If86aa5d54cfd7a7c32d630fb2bf4f47e057dbfb2 Reviewed-on: https://go-review.googlesource.com/115039 Reviewed-by: Ian Lance Taylor --- src/net/http/request.go | 3 ++- src/net/http/server.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/net/http/request.go b/src/net/http/request.go index 119a015a53d1d..7c4325027cf2d 100644 --- a/src/net/http/request.go +++ b/src/net/http/request.go @@ -316,7 +316,8 @@ type Request struct { // // For incoming server requests, the context is canceled when the // client's connection closes, the request is canceled (with HTTP/2), -// or when the ServeHTTP method returns. +// the ServeHTTP method returns, or if the Hijack method is +// called on the ResponseWriter. func (r *Request) Context() context.Context { if r.ctx != nil { return r.ctx diff --git a/src/net/http/server.go b/src/net/http/server.go index ac5cadd8d0a54..2d3486dc66083 100644 --- a/src/net/http/server.go +++ b/src/net/http/server.go @@ -187,8 +187,8 @@ type Hijacker interface { // The returned bufio.Reader may contain unprocessed buffered // data from the client. // - // After a call to Hijack, the original Request.Body must - // not be used. + // After a call to Hijack, the original Request.Body must not + // be used, and the Request.Context will be canceled. Hijack() (net.Conn, *bufio.ReadWriter, error) } From 555eb70db2f7ff4aa22768339425430c1dfdf6ac Mon Sep 17 00:00:00 2001 From: Tim Cooper Date: Tue, 29 May 2018 17:29:31 -0300 Subject: [PATCH 021/187] all: regenerate stringer files Change-Id: I34838320047792c4719837591e848b87ccb7f5ab Reviewed-on: https://go-review.googlesource.com/115058 Reviewed-by: Brad Fitzpatrick --- src/cmd/internal/objabi/reloctype_string.go | 4 ++-- src/debug/macho/reloctype_string.go | 10 +++++----- src/math/big/accuracy_string.go | 8 ++++---- src/math/big/roundingmode_string.go | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/cmd/internal/objabi/reloctype_string.go b/src/cmd/internal/objabi/reloctype_string.go index b3dbaa39eb27c..2cd3a9404546a 100644 --- a/src/cmd/internal/objabi/reloctype_string.go +++ b/src/cmd/internal/objabi/reloctype_string.go @@ -4,9 +4,9 @@ package objabi import "strconv" -const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFF" +const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORT" -var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 129, 136, 144, 152, 160, 166, 172, 178, 188, 197, 208, 219, 229, 238, 251, 265, 279, 293, 309, 323, 337, 348, 362, 377, 394, 412, 433, 443, 454, 467, 478} +var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 129, 136, 144, 152, 160, 166, 172, 178, 188, 197, 208, 219, 229, 238, 251, 265, 279, 293, 309, 323, 337, 348, 362, 377, 394, 412, 433, 443, 454, 467, 478, 490} func (i RelocType) String() string { i -= 1 diff --git a/src/debug/macho/reloctype_string.go b/src/debug/macho/reloctype_string.go index 6d5c5d87e82c5..9c2b13186e7d6 100644 --- a/src/debug/macho/reloctype_string.go +++ b/src/debug/macho/reloctype_string.go @@ -2,7 +2,7 @@ package macho -import "fmt" +import "strconv" const _RelocTypeGeneric_name = "GENERIC_RELOC_VANILLAGENERIC_RELOC_PAIRGENERIC_RELOC_SECTDIFFGENERIC_RELOC_PB_LA_PTRGENERIC_RELOC_LOCAL_SECTDIFFGENERIC_RELOC_TLV" @@ -10,7 +10,7 @@ var _RelocTypeGeneric_index = [...]uint8{0, 21, 39, 61, 84, 112, 129} func (i RelocTypeGeneric) String() string { if i < 0 || i >= RelocTypeGeneric(len(_RelocTypeGeneric_index)-1) { - return fmt.Sprintf("RelocTypeGeneric(%d)", i) + return "RelocTypeGeneric(" + strconv.FormatInt(int64(i), 10) + ")" } return _RelocTypeGeneric_name[_RelocTypeGeneric_index[i]:_RelocTypeGeneric_index[i+1]] } @@ -21,7 +21,7 @@ var _RelocTypeX86_64_index = [...]uint8{0, 21, 40, 59, 80, 96, 119, 140, 161, 18 func (i RelocTypeX86_64) String() string { if i < 0 || i >= RelocTypeX86_64(len(_RelocTypeX86_64_index)-1) { - return fmt.Sprintf("RelocTypeX86_64(%d)", i) + return "RelocTypeX86_64(" + strconv.FormatInt(int64(i), 10) + ")" } return _RelocTypeX86_64_name[_RelocTypeX86_64_index[i]:_RelocTypeX86_64_index[i+1]] } @@ -32,7 +32,7 @@ var _RelocTypeARM_index = [...]uint8{0, 17, 31, 49, 73, 92, 106, 126, 148, 162, func (i RelocTypeARM) String() string { if i < 0 || i >= RelocTypeARM(len(_RelocTypeARM_index)-1) { - return fmt.Sprintf("RelocTypeARM(%d)", i) + return "RelocTypeARM(" + strconv.FormatInt(int64(i), 10) + ")" } return _RelocTypeARM_name[_RelocTypeARM_index[i]:_RelocTypeARM_index[i+1]] } @@ -43,7 +43,7 @@ var _RelocTypeARM64_index = [...]uint16{0, 20, 42, 62, 80, 101, 128, 158, 184, 2 func (i RelocTypeARM64) String() string { if i < 0 || i >= RelocTypeARM64(len(_RelocTypeARM64_index)-1) { - return fmt.Sprintf("RelocTypeARM64(%d)", i) + return "RelocTypeARM64(" + strconv.FormatInt(int64(i), 10) + ")" } return _RelocTypeARM64_name[_RelocTypeARM64_index[i]:_RelocTypeARM64_index[i+1]] } diff --git a/src/math/big/accuracy_string.go b/src/math/big/accuracy_string.go index 24ef7f1077002..1501ace00d405 100644 --- a/src/math/big/accuracy_string.go +++ b/src/math/big/accuracy_string.go @@ -1,8 +1,8 @@ -// generated by stringer -type=Accuracy; DO NOT EDIT +// Code generated by "stringer -type=Accuracy"; DO NOT EDIT. package big -import "fmt" +import "strconv" const _Accuracy_name = "BelowExactAbove" @@ -10,8 +10,8 @@ var _Accuracy_index = [...]uint8{0, 5, 10, 15} func (i Accuracy) String() string { i -= -1 - if i < 0 || i+1 >= Accuracy(len(_Accuracy_index)) { - return fmt.Sprintf("Accuracy(%d)", i+-1) + if i < 0 || i >= Accuracy(len(_Accuracy_index)-1) { + return "Accuracy(" + strconv.FormatInt(int64(i+-1), 10) + ")" } return _Accuracy_name[_Accuracy_index[i]:_Accuracy_index[i+1]] } diff --git a/src/math/big/roundingmode_string.go b/src/math/big/roundingmode_string.go index 05024b806562e..c7629eb98b695 100644 --- a/src/math/big/roundingmode_string.go +++ b/src/math/big/roundingmode_string.go @@ -1,16 +1,16 @@ -// generated by stringer -type=RoundingMode; DO NOT EDIT +// Code generated by "stringer -type=RoundingMode"; DO NOT EDIT. package big -import "fmt" +import "strconv" const _RoundingMode_name = "ToNearestEvenToNearestAwayToZeroAwayFromZeroToNegativeInfToPositiveInf" var _RoundingMode_index = [...]uint8{0, 13, 26, 32, 44, 57, 70} func (i RoundingMode) String() string { - if i+1 >= RoundingMode(len(_RoundingMode_index)) { - return fmt.Sprintf("RoundingMode(%d)", i) + if i >= RoundingMode(len(_RoundingMode_index)-1) { + return "RoundingMode(" + strconv.FormatInt(int64(i), 10) + ")" } return _RoundingMode_name[_RoundingMode_index[i]:_RoundingMode_index[i+1]] } From 79fbe92b7e834dd3769807647718bdeb24f0a9d2 Mon Sep 17 00:00:00 2001 From: Tim Cooper Date: Sat, 21 Apr 2018 08:20:39 -0300 Subject: [PATCH 022/187] os/signal: remove unnecessary else condition Change-Id: I00f0195d54bf9bc30073741974ab941ec4d51a5c Reviewed-on: https://go-review.googlesource.com/108635 Reviewed-by: Brad Fitzpatrick --- src/os/signal/signal.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/os/signal/signal.go b/src/os/signal/signal.go index dc6b674c4f82c..a0eba0d50fb45 100644 --- a/src/os/signal/signal.go +++ b/src/os/signal/signal.go @@ -88,11 +88,8 @@ func Ignore(sig ...os.Signal) { // Ignored reports whether sig is currently ignored. func Ignored(sig os.Signal) bool { - if sn := signum(sig); sn < 0 { - return false - } else { - return signalIgnored(sn) - } + sn := signum(sig) + return sn >= 0 && signalIgnored(sn) } // Notify causes package signal to relay incoming signals to c. From b28e33d371cf96f6f34bbc576b6fd95468ea25bc Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Tue, 29 May 2018 13:13:24 -0700 Subject: [PATCH 023/187] go/types: add struct field with invalid type if field has errors This ensures that all struct fields are present and thus the struct has the original number of fields even if some fields have type errors. (This only applies as long as the field names themselves don't conflict.) Fixes #25627. Change-Id: I2414b1f432ce139b3cd2776ff0d46d8dcf38b650 Reviewed-on: https://go-review.googlesource.com/115115 Reviewed-by: Alan Donovan --- src/go/types/issues_test.go | 41 ++++++++++++++++++++++++++++++++ src/go/types/testdata/decls3.src | 4 ++-- src/go/types/typexpr.go | 17 +++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/go/types/issues_test.go b/src/go/types/issues_test.go index 02af0cf51b12d..8560bb9b7dc8d 100644 --- a/src/go/types/issues_test.go +++ b/src/go/types/issues_test.go @@ -314,3 +314,44 @@ func TestIssue22525(t *testing.T) { t.Errorf("got: %swant: %s", got, want) } } + +func TestIssue25627(t *testing.T) { + const prefix = `package p; import "unsafe"; type P *struct{}; type I interface{}; type T ` + // The src strings (without prefix) are constructed such that the number of semicolons + // plus one corresponds to the number of fields expected in the respective struct. + for _, src := range []string{ + `struct { x Missing }`, + `struct { Missing }`, + `struct { *Missing }`, + `struct { unsafe.Pointer }`, + `struct { P }`, + `struct { *I }`, + `struct { a int; b Missing; *Missing }`, + } { + f, err := parser.ParseFile(fset, "", prefix+src, 0) + if err != nil { + t.Fatal(err) + } + + cfg := Config{Importer: importer.Default(), Error: func(err error) {}} + info := &Info{Types: make(map[ast.Expr]TypeAndValue)} + _, err = cfg.Check(f.Name.Name, fset, []*ast.File{f}, info) + if err != nil { + if _, ok := err.(Error); !ok { + t.Fatal(err) + } + } + + ast.Inspect(f, func(n ast.Node) bool { + if spec, _ := n.(*ast.TypeSpec); spec != nil { + if tv, ok := info.Types[spec.Type]; ok && spec.Name.Name == "T" { + want := strings.Count(src, ";") + 1 + if got := tv.Type.(*Struct).NumFields(); got != want { + t.Errorf("%s: got %d fields; want %d", src, got, want) + } + } + } + return true + }) + } +} diff --git a/src/go/types/testdata/decls3.src b/src/go/types/testdata/decls3.src index 3071fdae5eecc..18ddf5859c602 100644 --- a/src/go/types/testdata/decls3.src +++ b/src/go/types/testdata/decls3.src @@ -99,9 +99,9 @@ func _() { // unsafe.Pointers are treated like regular pointers when embedded type T2 struct { unsafe /* ERROR "cannot be unsafe.Pointer" */ .Pointer - */* ERROR "cannot be unsafe.Pointer" */ unsafe.Pointer + */* ERROR "cannot be unsafe.Pointer" */ /* ERROR "Pointer redeclared" */ unsafe.Pointer UP /* ERROR "cannot be unsafe.Pointer" */ - * /* ERROR "cannot be unsafe.Pointer" */ UP + * /* ERROR "cannot be unsafe.Pointer" */ /* ERROR "UP redeclared" */ UP } } diff --git a/src/go/types/typexpr.go b/src/go/types/typexpr.go index 999383ed275a7..d3841c93676db 100644 --- a/src/go/types/typexpr.go +++ b/src/go/types/typexpr.go @@ -677,6 +677,16 @@ func (check *Checker) structType(styp *Struct, e *ast.StructType, path []*TypeNa } } + // addInvalid adds an embedded field of invalid type to the struct for + // fields with errors; this keeps the number of struct fields in sync + // with the source as long as the fields are _ or have different names + // (issue #25627). + addInvalid := func(ident *ast.Ident, pos token.Pos) { + typ = Typ[Invalid] + tag = "" + add(ident, true, pos) + } + for _, f := range list.List { typ = check.typExpr(f.Type, nil, path) tag = check.tag(f.Tag) @@ -693,6 +703,9 @@ func (check *Checker) structType(styp *Struct, e *ast.StructType, path []*TypeNa name := embeddedFieldIdent(f.Type) if name == nil { check.invalidAST(pos, "embedded field type %s has no name", f.Type) + name = ast.NewIdent("_") + name.NamePos = pos + addInvalid(name, pos) continue } t, isPtr := deref(typ) @@ -702,22 +715,26 @@ func (check *Checker) structType(styp *Struct, e *ast.StructType, path []*TypeNa case *Basic: if t == Typ[Invalid] { // error was reported before + addInvalid(name, pos) continue } // unsafe.Pointer is treated like a regular pointer if t.kind == UnsafePointer { check.errorf(pos, "embedded field type cannot be unsafe.Pointer") + addInvalid(name, pos) continue } case *Pointer: check.errorf(pos, "embedded field type cannot be a pointer") + addInvalid(name, pos) continue case *Interface: if isPtr { check.errorf(pos, "embedded field type cannot be a pointer to an interface") + addInvalid(name, pos) continue } } From 210a9e0c7dbe9bc16522387e7a0c902d29a5f85c Mon Sep 17 00:00:00 2001 From: Roger Peppe Date: Wed, 4 Oct 2017 19:28:59 +0100 Subject: [PATCH 024/187] net/http: vendor x/net/http/httpproxy, use it in net/http From x/net git rev c7086645de2. Updates #16704 Change-Id: I4d642478fc69a52c973964845fca2fd402716e57 Reviewed-on: https://go-review.googlesource.com/68091 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/go/build/deps_test.go | 1 + src/net/http/export_test.go | 4 +- src/net/http/proxy_test.go | 39 +-- src/net/http/transport.go | 141 +-------- src/net/http/transport_test.go | 60 ++-- .../x/net/http/httpproxy/export_test.go | 7 + .../golang_org/x/net/http/httpproxy/proxy.go | 239 ++++++++++++++ .../x/net/http/httpproxy/proxy_test.go | 298 ++++++++++++++++++ 8 files changed, 605 insertions(+), 184 deletions(-) create mode 100644 src/vendor/golang_org/x/net/http/httpproxy/export_test.go create mode 100644 src/vendor/golang_org/x/net/http/httpproxy/proxy.go create mode 100644 src/vendor/golang_org/x/net/http/httpproxy/proxy_test.go diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index 9aebd7327a9da..5137ccfe3f8ae 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -401,6 +401,7 @@ var pkgDeps = map[string][]string{ "crypto/rand", "crypto/tls", "golang_org/x/net/http/httpguts", + "golang_org/x/net/http/httpproxy", "golang_org/x/net/http2/hpack", "golang_org/x/net/idna", "golang_org/x/text/unicode/norm", diff --git a/src/net/http/export_test.go b/src/net/http/export_test.go index 1825acd9be7a3..e0ceb40021c0f 100644 --- a/src/net/http/export_test.go +++ b/src/net/http/export_test.go @@ -76,9 +76,7 @@ func NewTestTimeoutHandler(handler Handler, ch <-chan time.Time) Handler { } func ResetCachedEnvironment() { - httpProxyEnv.reset() - httpsProxyEnv.reset() - noProxyEnv.reset() + resetProxyConfig() } func (t *Transport) NumPendingRequestsForTesting() int { diff --git a/src/net/http/proxy_test.go b/src/net/http/proxy_test.go index f59a551f0acf4..eef0ca82f8c9a 100644 --- a/src/net/http/proxy_test.go +++ b/src/net/http/proxy_test.go @@ -13,37 +13,6 @@ import ( // TODO(mattn): // test ProxyAuth -var UseProxyTests = []struct { - host string - match bool -}{ - // Never proxy localhost: - {"localhost", false}, - {"127.0.0.1", false}, - {"127.0.0.2", false}, - {"[::1]", false}, - {"[::2]", true}, // not a loopback address - - {"barbaz.net", false}, // match as .barbaz.net - {"foobar.com", false}, // have a port but match - {"foofoobar.com", true}, // not match as a part of foobar.com - {"baz.com", true}, // not match as a part of barbaz.com - {"localhost.net", true}, // not match as suffix of address - {"local.localhost", true}, // not match as prefix as address - {"barbarbaz.net", true}, // not match because NO_PROXY have a '.' - {"www.foobar.com", false}, // match because NO_PROXY includes "foobar.com" -} - -func TestUseProxy(t *testing.T) { - ResetProxyEnv() - os.Setenv("NO_PROXY", "foobar.com, .barbaz.net") - for _, test := range UseProxyTests { - if useProxy(test.host+":80") != test.match { - t.Errorf("useProxy(%v) = %v, want %v", test.host, !test.match, test.match) - } - } -} - var cacheKeysTests = []struct { proxy string scheme string @@ -74,14 +43,8 @@ func TestCacheKeys(t *testing.T) { } func ResetProxyEnv() { - for _, v := range []string{"HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy"} { + for _, v := range []string{"HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy", "REQUEST_METHOD"} { os.Unsetenv(v) } ResetCachedEnvironment() } - -func TestInvalidNoProxy(t *testing.T) { - ResetProxyEnv() - os.Setenv("NO_PROXY", ":1") - useProxy("example.com:80") // should not panic -} diff --git a/src/net/http/transport.go b/src/net/http/transport.go index cce88ca239ac2..5bf9ff951f52c 100644 --- a/src/net/http/transport.go +++ b/src/net/http/transport.go @@ -29,6 +29,7 @@ import ( "time" "golang_org/x/net/http/httpguts" + "golang_org/x/net/http/httpproxy" ) // DefaultTransport is the default implementation of Transport and is @@ -272,39 +273,7 @@ func (t *Transport) onceSetNextProtoDefaults() { // As a special case, if req.URL.Host is "localhost" (with or without // a port number), then a nil URL and nil error will be returned. func ProxyFromEnvironment(req *Request) (*url.URL, error) { - var proxy string - if req.URL.Scheme == "https" { - proxy = httpsProxyEnv.Get() - } - if proxy == "" { - proxy = httpProxyEnv.Get() - if proxy != "" && os.Getenv("REQUEST_METHOD") != "" { - return nil, errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") - } - } - if proxy == "" { - return nil, nil - } - if !useProxy(canonicalAddr(req.URL)) { - return nil, nil - } - proxyURL, err := url.Parse(proxy) - if err != nil || - (proxyURL.Scheme != "http" && - proxyURL.Scheme != "https" && - proxyURL.Scheme != "socks5") { - // proxy was bogus. Try prepending "http://" to it and - // see if that parses correctly. If not, we fall - // through and complain about the original one. - if proxyURL, err := url.Parse("http://" + proxy); err == nil { - return proxyURL, nil - } - - } - if err != nil { - return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) - } - return proxyURL, nil + return envProxyFunc()(req.URL) } // ProxyURL returns a proxy function (for use in a Transport) @@ -574,44 +543,25 @@ func (t *Transport) cancelRequest(req *Request, err error) { // var ( - httpProxyEnv = &envOnce{ - names: []string{"HTTP_PROXY", "http_proxy"}, - } - httpsProxyEnv = &envOnce{ - names: []string{"HTTPS_PROXY", "https_proxy"}, - } - noProxyEnv = &envOnce{ - names: []string{"NO_PROXY", "no_proxy"}, - } + // proxyConfigOnce guards proxyConfig + envProxyOnce sync.Once + envProxyFuncValue func(*url.URL) (*url.URL, error) ) -// envOnce looks up an environment variable (optionally by multiple -// names) once. It mitigates expensive lookups on some platforms -// (e.g. Windows). -type envOnce struct { - names []string - once sync.Once - val string +// defaultProxyConfig returns a ProxyConfig value looked up +// from the environment. This mitigates expensive lookups +// on some platforms (e.g. Windows). +func envProxyFunc() func(*url.URL) (*url.URL, error) { + envProxyOnce.Do(func() { + envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() + }) + return envProxyFuncValue } -func (e *envOnce) Get() string { - e.once.Do(e.init) - return e.val -} - -func (e *envOnce) init() { - for _, n := range e.names { - e.val = os.Getenv(n) - if e.val != "" { - return - } - } -} - -// reset is used by tests -func (e *envOnce) reset() { - e.once = sync.Once{} - e.val = "" +// resetProxyConfig is used by tests. +func resetProxyConfig() { + envProxyOnce = sync.Once{} + envProxyFuncValue = nil } func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { @@ -1235,63 +1185,6 @@ func (w persistConnWriter) Write(p []byte) (n int, err error) { return } -// useProxy reports whether requests to addr should use a proxy, -// according to the NO_PROXY or no_proxy environment variable. -// addr is always a canonicalAddr with a host and port. -func useProxy(addr string) bool { - if len(addr) == 0 { - return true - } - host, _, err := net.SplitHostPort(addr) - if err != nil { - return false - } - if host == "localhost" { - return false - } - if ip := net.ParseIP(host); ip != nil { - if ip.IsLoopback() { - return false - } - } - - noProxy := noProxyEnv.Get() - if noProxy == "*" { - return false - } - - addr = strings.ToLower(strings.TrimSpace(addr)) - if hasPort(addr) { - addr = addr[:strings.LastIndex(addr, ":")] - } - - for _, p := range strings.Split(noProxy, ",") { - p = strings.ToLower(strings.TrimSpace(p)) - if len(p) == 0 { - continue - } - if hasPort(p) { - p = p[:strings.LastIndex(p, ":")] - } - if addr == p { - return false - } - if len(p) == 0 { - // There is no host part, likely the entry is malformed; ignore. - continue - } - if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { - // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com" - return false - } - if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { - // no_proxy "foo.com" matches "bar.foo.com" - return false - } - } - return true -} - // connectMethod is the map key (in its String form) for keeping persistent // TCP connections alive for subsequent HTTP requests. // diff --git a/src/net/http/transport_test.go b/src/net/http/transport_test.go index 5e35812c7b989..57309bbac1fd5 100644 --- a/src/net/http/transport_test.go +++ b/src/net/http/transport_test.go @@ -2381,7 +2381,7 @@ var proxyFromEnvTests = []proxyFromEnvTest{ // where HTTP_PROXY can be attacker-controlled. {env: "http://10.1.2.3:8080", reqmeth: "POST", want: "", - wanterr: errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")}, + wanterr: errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")}, {want: ""}, @@ -2392,28 +2392,50 @@ var proxyFromEnvTests = []proxyFromEnvTest{ {noenv: ".foo.com", req: "http://example.com/", env: "proxy", want: "http://proxy"}, } +func testProxyForRequest(t *testing.T, tt proxyFromEnvTest, proxyForRequest func(req *Request) (*url.URL, error)) { + t.Helper() + reqURL := tt.req + if reqURL == "" { + reqURL = "http://example.com" + } + req, _ := NewRequest("GET", reqURL, nil) + url, err := proxyForRequest(req) + if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.wanterr); g != e { + t.Errorf("%v: got error = %q, want %q", tt, g, e) + return + } + if got := fmt.Sprintf("%s", url); got != tt.want { + t.Errorf("%v: got URL = %q, want %q", tt, url, tt.want) + } +} + func TestProxyFromEnvironment(t *testing.T) { ResetProxyEnv() defer ResetProxyEnv() for _, tt := range proxyFromEnvTests { - os.Setenv("HTTP_PROXY", tt.env) - os.Setenv("HTTPS_PROXY", tt.httpsenv) - os.Setenv("NO_PROXY", tt.noenv) - os.Setenv("REQUEST_METHOD", tt.reqmeth) - ResetCachedEnvironment() - reqURL := tt.req - if reqURL == "" { - reqURL = "http://example.com" - } - req, _ := NewRequest("GET", reqURL, nil) - url, err := ProxyFromEnvironment(req) - if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.wanterr); g != e { - t.Errorf("%v: got error = %q, want %q", tt, g, e) - continue - } - if got := fmt.Sprintf("%s", url); got != tt.want { - t.Errorf("%v: got URL = %q, want %q", tt, url, tt.want) - } + testProxyForRequest(t, tt, func(req *Request) (*url.URL, error) { + os.Setenv("HTTP_PROXY", tt.env) + os.Setenv("HTTPS_PROXY", tt.httpsenv) + os.Setenv("NO_PROXY", tt.noenv) + os.Setenv("REQUEST_METHOD", tt.reqmeth) + ResetCachedEnvironment() + return ProxyFromEnvironment(req) + }) + } +} + +func TestProxyFromEnvironmentLowerCase(t *testing.T) { + ResetProxyEnv() + defer ResetProxyEnv() + for _, tt := range proxyFromEnvTests { + testProxyForRequest(t, tt, func(req *Request) (*url.URL, error) { + os.Setenv("http_proxy", tt.env) + os.Setenv("https_proxy", tt.httpsenv) + os.Setenv("no_proxy", tt.noenv) + os.Setenv("REQUEST_METHOD", tt.reqmeth) + ResetCachedEnvironment() + return ProxyFromEnvironment(req) + }) } } diff --git a/src/vendor/golang_org/x/net/http/httpproxy/export_test.go b/src/vendor/golang_org/x/net/http/httpproxy/export_test.go new file mode 100644 index 0000000000000..36b29d2db669f --- /dev/null +++ b/src/vendor/golang_org/x/net/http/httpproxy/export_test.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httpproxy + +var ExportUseProxy = (*Config).useProxy diff --git a/src/vendor/golang_org/x/net/http/httpproxy/proxy.go b/src/vendor/golang_org/x/net/http/httpproxy/proxy.go new file mode 100644 index 0000000000000..f82748d208a82 --- /dev/null +++ b/src/vendor/golang_org/x/net/http/httpproxy/proxy.go @@ -0,0 +1,239 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpproxy provides support for HTTP proxy determination +// based on environment variables, as provided by net/http's +// ProxyFromEnvironment function. +// +// The API is not subject to the Go 1 compatibility promise and may change at +// any time. +package httpproxy + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "unicode/utf8" + + "golang_org/x/net/idna" +) + +// Config holds configuration for HTTP proxy settings. See +// FromEnvironment for details. +type Config struct { + // HTTPProxy represents the value of the HTTP_PROXY or + // http_proxy environment variable. It will be used as the proxy + // URL for HTTP requests and HTTPS requests unless overridden by + // HTTPSProxy or NoProxy. + HTTPProxy string + + // HTTPSProxy represents the HTTPS_PROXY or https_proxy + // environment variable. It will be used as the proxy URL for + // HTTPS requests unless overridden by NoProxy. + HTTPSProxy string + + // NoProxy represents the NO_PROXY or no_proxy environment + // variable. It specifies URLs that should be excluded from + // proxying as a comma-separated list of domain names or a + // single asterisk (*) to indicate that no proxying should be + // done. A domain name matches that name and all subdomains. A + // domain name with a leading "." matches subdomains only. For + // example "foo.com" matches "foo.com" and "bar.foo.com"; + // ".y.com" matches "x.y.com" but not "y.com". + NoProxy string + + // CGI holds whether the current process is running + // as a CGI handler (FromEnvironment infers this from the + // presence of a REQUEST_METHOD environment variable). + // When this is set, ProxyForURL will return an error + // when HTTPProxy applies, because a client could be + // setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy. + CGI bool +} + +// FromEnvironment returns a Config instance populated from the +// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the +// lowercase versions thereof). HTTPS_PROXY takes precedence over +// HTTP_PROXY for https requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. An error +// is returned if the value is a different form. +func FromEnvironment() *Config { + return &Config{ + HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"), + HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"), + NoProxy: getEnvAny("NO_PROXY", "no_proxy"), + CGI: os.Getenv("REQUEST_METHOD") != "", + } +} + +func getEnvAny(names ...string) string { + for _, n := range names { + if val := os.Getenv(n); val != "" { + return val + } + } + return "" +} + +// ProxyFunc returns a function that determines the proxy URL to use for +// a given request URL. Changing the contents of cfg will not affect +// proxy functions created earlier. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, as +// defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without a +// port number), then a nil URL and nil error will be returned. +func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) { + // Prevent Config changes from affecting the function calculation. + // TODO Preprocess proxy settings for more efficient evaluation. + cfg1 := *cfg + return cfg1.proxyForURL +} + +func (cfg *Config) proxyForURL(reqURL *url.URL) (*url.URL, error) { + var proxy string + if reqURL.Scheme == "https" { + proxy = cfg.HTTPSProxy + } + if proxy == "" { + proxy = cfg.HTTPProxy + if proxy != "" && cfg.CGI { + return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } + } + if proxy == "" { + return nil, nil + } + if !cfg.useProxy(canonicalAddr(reqURL)) { + return nil, nil + } + proxyURL, err := url.Parse(proxy) + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func (cfg *Config) useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() { + return false + } + } + + noProxy := cfg.NoProxy + if noProxy == "*" { + return false + } + + addr = strings.ToLower(strings.TrimSpace(addr)) + if hasPort(addr) { + addr = addr[:strings.LastIndex(addr, ":")] + } + + for _, p := range strings.Split(noProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + if hasPort(p) { + p = p[:strings.LastIndex(p, ":")] + } + if addr == p { + return false + } + if len(p) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { + // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com" + return false + } + if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { + // no_proxy "foo.com" matches "bar.foo.com" + return false + } + } + return true +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if isASCII(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} diff --git a/src/vendor/golang_org/x/net/http/httpproxy/proxy_test.go b/src/vendor/golang_org/x/net/http/httpproxy/proxy_test.go new file mode 100644 index 0000000000000..fde2514832bab --- /dev/null +++ b/src/vendor/golang_org/x/net/http/httpproxy/proxy_test.go @@ -0,0 +1,298 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httpproxy_test + +import ( + "bytes" + "errors" + "fmt" + "net/url" + "os" + "strings" + "testing" + + "golang_org/x/net/http/httpproxy" +) + +type proxyForURLTest struct { + cfg httpproxy.Config + req string // URL to fetch; blank means "http://example.com" + want string + wanterr error +} + +func (t proxyForURLTest) String() string { + var buf bytes.Buffer + space := func() { + if buf.Len() > 0 { + buf.WriteByte(' ') + } + } + if t.cfg.HTTPProxy != "" { + fmt.Fprintf(&buf, "http_proxy=%q", t.cfg.HTTPProxy) + } + if t.cfg.HTTPSProxy != "" { + space() + fmt.Fprintf(&buf, "https_proxy=%q", t.cfg.HTTPSProxy) + } + if t.cfg.NoProxy != "" { + space() + fmt.Fprintf(&buf, "no_proxy=%q", t.cfg.NoProxy) + } + req := "http://example.com" + if t.req != "" { + req = t.req + } + space() + fmt.Fprintf(&buf, "req=%q", req) + return strings.TrimSpace(buf.String()) +} + +var proxyForURLTests = []proxyForURLTest{{ + cfg: httpproxy.Config{ + HTTPProxy: "127.0.0.1:8080", + }, + want: "http://127.0.0.1:8080", +}, { + cfg: httpproxy.Config{ + HTTPProxy: "cache.corp.example.com:1234", + }, + want: "http://cache.corp.example.com:1234", +}, { + cfg: httpproxy.Config{ + HTTPProxy: "cache.corp.example.com", + }, + want: "http://cache.corp.example.com", +}, { + cfg: httpproxy.Config{ + HTTPProxy: "https://cache.corp.example.com", + }, + want: "https://cache.corp.example.com", +}, { + cfg: httpproxy.Config{ + HTTPProxy: "http://127.0.0.1:8080", + }, + want: "http://127.0.0.1:8080", +}, { + cfg: httpproxy.Config{ + HTTPProxy: "https://127.0.0.1:8080", + }, + want: "https://127.0.0.1:8080", +}, { + cfg: httpproxy.Config{ + HTTPProxy: "socks5://127.0.0.1", + }, + want: "socks5://127.0.0.1", +}, { + // Don't use secure for http + cfg: httpproxy.Config{ + HTTPProxy: "http.proxy.tld", + HTTPSProxy: "secure.proxy.tld", + }, + req: "http://insecure.tld/", + want: "http://http.proxy.tld", +}, { + // Use secure for https. + cfg: httpproxy.Config{ + HTTPProxy: "http.proxy.tld", + HTTPSProxy: "secure.proxy.tld", + }, + req: "https://secure.tld/", + want: "http://secure.proxy.tld", +}, { + cfg: httpproxy.Config{ + HTTPProxy: "http.proxy.tld", + HTTPSProxy: "https://secure.proxy.tld", + }, + req: "https://secure.tld/", + want: "https://secure.proxy.tld", +}, { + // Issue 16405: don't use HTTP_PROXY in a CGI environment, + // where HTTP_PROXY can be attacker-controlled. + cfg: httpproxy.Config{ + HTTPProxy: "http://10.1.2.3:8080", + CGI: true, + }, + want: "", + wanterr: errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy"), +}, { + // HTTPS proxy is still used even in CGI environment. + // (perhaps dubious but it's the historical behaviour). + cfg: httpproxy.Config{ + HTTPSProxy: "https://secure.proxy.tld", + CGI: true, + }, + req: "https://secure.tld/", + want: "https://secure.proxy.tld", +}, { + want: "", +}, { + cfg: httpproxy.Config{ + NoProxy: "example.com", + HTTPProxy: "proxy", + }, + req: "http://example.com/", + want: "", +}, { + cfg: httpproxy.Config{ + NoProxy: ".example.com", + HTTPProxy: "proxy", + }, + req: "http://example.com/", + want: "", +}, { + cfg: httpproxy.Config{ + NoProxy: "ample.com", + HTTPProxy: "proxy", + }, + req: "http://example.com/", + want: "http://proxy", +}, { + cfg: httpproxy.Config{ + NoProxy: "example.com", + HTTPProxy: "proxy", + }, + req: "http://foo.example.com/", + want: "", +}, { + cfg: httpproxy.Config{ + NoProxy: ".foo.com", + HTTPProxy: "proxy", + }, + req: "http://example.com/", + want: "http://proxy", +}} + +func testProxyForURL(t *testing.T, tt proxyForURLTest) { + t.Helper() + reqURLStr := tt.req + if reqURLStr == "" { + reqURLStr = "http://example.com" + } + reqURL, err := url.Parse(reqURLStr) + if err != nil { + t.Errorf("invalid URL %q", reqURLStr) + return + } + cfg := tt.cfg + proxyForURL := cfg.ProxyFunc() + url, err := proxyForURL(reqURL) + if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.wanterr); g != e { + t.Errorf("%v: got error = %q, want %q", tt, g, e) + return + } + if got := fmt.Sprintf("%s", url); got != tt.want { + t.Errorf("%v: got URL = %q, want %q", tt, url, tt.want) + } + + // Check that changing the Config doesn't change the results + // of the functuon. + cfg = httpproxy.Config{} + url, err = proxyForURL(reqURL) + if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.wanterr); g != e { + t.Errorf("(after mutating config) %v: got error = %q, want %q", tt, g, e) + return + } + if got := fmt.Sprintf("%s", url); got != tt.want { + t.Errorf("(after mutating config) %v: got URL = %q, want %q", tt, url, tt.want) + } +} + +func TestProxyForURL(t *testing.T) { + for _, tt := range proxyForURLTests { + testProxyForURL(t, tt) + } +} + +func TestFromEnvironment(t *testing.T) { + os.Setenv("HTTP_PROXY", "httpproxy") + os.Setenv("HTTPS_PROXY", "httpsproxy") + os.Setenv("NO_PROXY", "noproxy") + os.Setenv("REQUEST_METHOD", "") + got := httpproxy.FromEnvironment() + want := httpproxy.Config{ + HTTPProxy: "httpproxy", + HTTPSProxy: "httpsproxy", + NoProxy: "noproxy", + } + if *got != want { + t.Errorf("unexpected proxy config, got %#v want %#v", got, want) + } +} + +func TestFromEnvironmentWithRequestMethod(t *testing.T) { + os.Setenv("HTTP_PROXY", "httpproxy") + os.Setenv("HTTPS_PROXY", "httpsproxy") + os.Setenv("NO_PROXY", "noproxy") + os.Setenv("REQUEST_METHOD", "PUT") + got := httpproxy.FromEnvironment() + want := httpproxy.Config{ + HTTPProxy: "httpproxy", + HTTPSProxy: "httpsproxy", + NoProxy: "noproxy", + CGI: true, + } + if *got != want { + t.Errorf("unexpected proxy config, got %#v want %#v", got, want) + } +} + +func TestFromEnvironmentLowerCase(t *testing.T) { + os.Setenv("http_proxy", "httpproxy") + os.Setenv("https_proxy", "httpsproxy") + os.Setenv("no_proxy", "noproxy") + os.Setenv("REQUEST_METHOD", "") + got := httpproxy.FromEnvironment() + want := httpproxy.Config{ + HTTPProxy: "httpproxy", + HTTPSProxy: "httpsproxy", + NoProxy: "noproxy", + } + if *got != want { + t.Errorf("unexpected proxy config, got %#v want %#v", got, want) + } +} + +var UseProxyTests = []struct { + host string + match bool +}{ + // Never proxy localhost: + {"localhost", false}, + {"127.0.0.1", false}, + {"127.0.0.2", false}, + {"[::1]", false}, + {"[::2]", true}, // not a loopback address + + {"barbaz.net", false}, // match as .barbaz.net + {"foobar.com", false}, // have a port but match + {"foofoobar.com", true}, // not match as a part of foobar.com + {"baz.com", true}, // not match as a part of barbaz.com + {"localhost.net", true}, // not match as suffix of address + {"local.localhost", true}, // not match as prefix as address + {"barbarbaz.net", true}, // not match because NO_PROXY have a '.' + {"www.foobar.com", false}, // match because NO_PROXY includes "foobar.com" +} + +func TestUseProxy(t *testing.T) { + cfg := &httpproxy.Config{ + NoProxy: "foobar.com, .barbaz.net", + } + for _, test := range UseProxyTests { + if httpproxy.ExportUseProxy(cfg, test.host+":80") != test.match { + t.Errorf("useProxy(%v) = %v, want %v", test.host, !test.match, test.match) + } + } +} + +func TestInvalidNoProxy(t *testing.T) { + cfg := &httpproxy.Config{ + NoProxy: ":1", + } + ok := httpproxy.ExportUseProxy(cfg, "example.com:80") // should not panic + if !ok { + t.Errorf("useProxy unexpected return; got false; want true") + } +} From b0ac2546b1851a4835ad687e649dead7f610f6a9 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Thu, 15 Mar 2018 14:15:54 -0700 Subject: [PATCH 025/187] context: add benchmarks for context cancellation Change-Id: I539c9226eb7e493b52c50e1e431954567d43bcfb Reviewed-on: https://go-review.googlesource.com/100847 Reviewed-by: Brad Fitzpatrick --- src/context/benchmark_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/context/benchmark_test.go b/src/context/benchmark_test.go index 6dd8510ff4c43..5d56863050a9a 100644 --- a/src/context/benchmark_test.go +++ b/src/context/benchmark_test.go @@ -13,6 +13,30 @@ import ( "time" ) +func BenchmarkCommonParentCancel(b *testing.B) { + root := WithValue(Background(), "key", "value") + shared, sharedcancel := WithCancel(root) + defer sharedcancel() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + x := 0 + for pb.Next() { + ctx, cancel := WithCancel(shared) + if ctx.Value("key").(string) != "value" { + b.Fatal("should not be reached") + } + for i := 0; i < 100; i++ { + x /= x + 1 + } + cancel() + for i := 0; i < 100; i++ { + x /= x + 1 + } + } + }) +} + func BenchmarkWithTimeout(b *testing.B) { for concurrency := 40; concurrency <= 4e5; concurrency *= 100 { name := fmt.Sprintf("concurrency=%d", concurrency) From 6c6e22e5a9b70f22750e4fc210cd67175c6d1187 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 26 May 2018 09:58:26 -0700 Subject: [PATCH 026/187] runtime: implement time.now using libc Change-Id: Ibdd9202d9711ea8aab2446c9950ddb8e1f6bf4e0 Reviewed-on: https://go-review.googlesource.com/114799 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/vet/all/whitelist/darwin_386.txt | 2 - src/cmd/vet/all/whitelist/darwin_amd64.txt | 1 - src/runtime/sys_darwin.go | 10 ++ src/runtime/sys_darwin_386.s | 151 ++---------------- src/runtime/sys_darwin_amd64.s | 171 +-------------------- src/runtime/sys_darwin_arm.s | 22 +-- src/runtime/sys_darwin_arm64.s | 20 +-- src/runtime/timeasm.go | 2 +- src/runtime/timestub.go | 1 - src/runtime/timestub2.go | 2 +- 10 files changed, 36 insertions(+), 346 deletions(-) diff --git a/src/cmd/vet/all/whitelist/darwin_386.txt b/src/cmd/vet/all/whitelist/darwin_386.txt index f7645eff1a1e3..934b773f5081a 100644 --- a/src/cmd/vet/all/whitelist/darwin_386.txt +++ b/src/cmd/vet/all/whitelist/darwin_386.txt @@ -2,7 +2,5 @@ // Ok -runtime/sys_darwin_386.s: [386] now: function now missing Go declaration runtime/sys_darwin_386.s: [386] sysenter: function sysenter missing Go declaration runtime/sys_darwin_386.s: [386] setldt: function setldt missing Go declaration -runtime/sys_darwin_386.s: [386] cannot check cross-package assembly function: now is in package time diff --git a/src/cmd/vet/all/whitelist/darwin_amd64.txt b/src/cmd/vet/all/whitelist/darwin_amd64.txt index 8423415aea801..fcdacb2dc1b61 100644 --- a/src/cmd/vet/all/whitelist/darwin_amd64.txt +++ b/src/cmd/vet/all/whitelist/darwin_amd64.txt @@ -1,4 +1,3 @@ // darwin/amd64-specific vet whitelist. See readme.txt for details. runtime/sys_darwin_amd64.s: [amd64] settls: function settls missing Go declaration -runtime/sys_darwin_amd64.s: [amd64] cannot check cross-package assembly function: now is in package time diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 124b2fee5e22e..d8b5441b31743 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -170,6 +170,15 @@ func nanotime() int64 { } func nanotime_trampoline() +//go:nosplit +//go:cgo_unsafe_args +func walltime() (int64, int32) { + var t timeval + libcCall(unsafe.Pointer(funcPC(walltime_trampoline)), unsafe.Pointer(&t)) + return int64(t.tv_sec), 1000 * t.tv_usec +} +func walltime_trampoline() + // Not used on Darwin, but must be defined. func exitThread(wait *uint32) { } @@ -197,6 +206,7 @@ func exitThread(wait *uint32) { //go:cgo_import_dynamic libc_mach_timebase_info mach_timebase_info "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_mach_absolute_time mach_absolute_time "/usr/lib/libSystem.B.dylib" +//go:cgo_import_dynamic libc_gettimeofday gettimeofday "/usr/lib/libSystem.B.dylib" // Magic incantation to get libSystem actually dynamically linked. // TODO: Why does the code require this? See cmd/compile/internal/ld/go.go:210 diff --git a/src/runtime/sys_darwin_386.s b/src/runtime/sys_darwin_386.s index dc2b84c4845b1..5b29dfe6043f9 100644 --- a/src/runtime/sys_darwin_386.s +++ b/src/runtime/sys_darwin_386.s @@ -166,149 +166,16 @@ TEXT runtime·setitimer(SB),NOSPLIT,$0 INT $0x80 RET -// OS X comm page time offsets -// http://www.opensource.apple.com/source/xnu/xnu-1699.26.8/osfmk/i386/cpu_capabilities.h -#define cpu_capabilities 0x20 -#define nt_tsc_base 0x50 -#define nt_scale 0x58 -#define nt_shift 0x5c -#define nt_ns_base 0x60 -#define nt_generation 0x68 -#define gtod_generation 0x6c -#define gtod_ns_base 0x70 -#define gtod_sec_base 0x78 - -// called from assembly -// 64-bit unix nanoseconds returned in DX:AX. -// I'd much rather write this in C but we need -// assembly for the 96-bit multiply and RDTSC. -// -// Note that we could arrange to return monotonic time here -// as well, but we don't bother, for two reasons: -// 1. macOS only supports 64-bit systems, so no one should -// be using the 32-bit code in production. -// This code is only maintained to make it easier for developers -// using Macs to test the 32-bit compiler. -// 2. On some (probably now unsupported) CPUs, -// the code falls back to the system call always, -// so it can't even use the comm page at all. -TEXT runtime·now(SB),NOSPLIT,$40 - MOVL $0xffff0000, BP /* comm page base */ - - // Test for slow CPU. If so, the math is completely - // different, and unimplemented here, so use the - // system call. - MOVL cpu_capabilities(BP), AX - TESTL $0x4000, AX - JNZ systime - - // Loop trying to take a consistent snapshot - // of the time parameters. -timeloop: - MOVL gtod_generation(BP), BX - TESTL BX, BX - JZ systime - MOVL nt_generation(BP), CX - TESTL CX, CX - JZ timeloop - RDTSC - MOVL nt_tsc_base(BP), SI - MOVL (nt_tsc_base+4)(BP), DI - MOVL SI, 0(SP) - MOVL DI, 4(SP) - MOVL nt_scale(BP), SI - MOVL SI, 8(SP) - MOVL nt_ns_base(BP), SI - MOVL (nt_ns_base+4)(BP), DI - MOVL SI, 12(SP) - MOVL DI, 16(SP) - CMPL nt_generation(BP), CX - JNE timeloop - MOVL gtod_ns_base(BP), SI - MOVL (gtod_ns_base+4)(BP), DI - MOVL SI, 20(SP) - MOVL DI, 24(SP) - MOVL gtod_sec_base(BP), SI - MOVL (gtod_sec_base+4)(BP), DI - MOVL SI, 28(SP) - MOVL DI, 32(SP) - CMPL gtod_generation(BP), BX - JNE timeloop - - // Gathered all the data we need. Compute time. - // ((tsc - nt_tsc_base) * nt_scale) >> 32 + nt_ns_base - gtod_ns_base + gtod_sec_base*1e9 - // The multiply and shift extracts the top 64 bits of the 96-bit product. - SUBL 0(SP), AX // DX:AX = (tsc - nt_tsc_base) - SBBL 4(SP), DX - - // We have x = tsc - nt_tsc_base - DX:AX to be - // multiplied by y = nt_scale = 8(SP), keeping the top 64 bits of the 96-bit product. - // x*y = (x&0xffffffff)*y + (x&0xffffffff00000000)*y - // (x*y)>>32 = ((x&0xffffffff)*y)>>32 + (x>>32)*y - MOVL DX, CX // SI = (x&0xffffffff)*y >> 32 - MOVL $0, DX - MULL 8(SP) - MOVL DX, SI - - MOVL CX, AX // DX:AX = (x>>32)*y - MOVL $0, DX - MULL 8(SP) - - ADDL SI, AX // DX:AX += (x&0xffffffff)*y >> 32 - ADCL $0, DX - - // DX:AX is now ((tsc - nt_tsc_base) * nt_scale) >> 32. - ADDL 12(SP), AX // DX:AX += nt_ns_base - ADCL 16(SP), DX - SUBL 20(SP), AX // DX:AX -= gtod_ns_base - SBBL 24(SP), DX - MOVL AX, SI // DI:SI = DX:AX - MOVL DX, DI - MOVL 28(SP), AX // DX:AX = gtod_sec_base*1e9 - MOVL 32(SP), DX - MOVL $1000000000, CX - MULL CX - ADDL SI, AX // DX:AX += DI:SI - ADCL DI, DX - RET - -systime: - // Fall back to system call (usually first call in this thread) - LEAL 16(SP), AX // must be non-nil, unused - MOVL AX, 4(SP) - MOVL $0, 8(SP) // time zone pointer - MOVL $0, 12(SP) // required as of Sierra; Issue 16570 - MOVL $116, AX // SYS_GETTIMEOFDAY - INT $0x80 - CMPL AX, $0 - JNE inreg +TEXT runtime·walltime_trampoline(SB),NOSPLIT,$0 + PUSHL BP + MOVL SP, BP + SUBL $8, SP MOVL 16(SP), AX - MOVL 20(SP), DX -inreg: - // sec is in AX, usec in DX - // convert to DX:AX nsec - MOVL DX, BX - MOVL $1000000000, CX - MULL CX - IMULL $1000, BX - ADDL BX, AX - ADCL $0, DX - RET - -// func now() (sec int64, nsec int32, mono uint64) -TEXT time·now(SB),NOSPLIT,$0-20 - CALL runtime·now(SB) - MOVL AX, BX - MOVL DX, BP - SUBL runtime·startNano(SB), BX - SBBL runtime·startNano+4(SB), BP - MOVL BX, mono+12(FP) - MOVL BP, mono+16(FP) - MOVL $1000000000, CX - DIVL CX - MOVL AX, sec+0(FP) - MOVL $0, sec+4(FP) - MOVL DX, nsec+8(FP) + MOVL AX, 0(SP) // *timeval + MOVL $0, 4(SP) // no timezone needed + CALL libc_gettimeofday(SB) + MOVL BP, SP + POPL BP RET GLOBL timebase<>(SB),NOPTR,$(machTimebaseInfo__size) diff --git a/src/runtime/sys_darwin_amd64.s b/src/runtime/sys_darwin_amd64.s index 81684159328c4..320d56499ac5f 100644 --- a/src/runtime/sys_darwin_amd64.s +++ b/src/runtime/sys_darwin_amd64.s @@ -92,24 +92,6 @@ TEXT runtime·madvise_trampoline(SB), NOSPLIT, $0 POPQ BP RET -// OS X comm page time offsets -// https://opensource.apple.com/source/xnu/xnu-4570.1.46/osfmk/i386/cpu_capabilities.h - -#define nt_tsc_base 0x50 -#define nt_scale 0x58 -#define nt_shift 0x5c -#define nt_ns_base 0x60 -#define nt_generation 0x68 -#define gtod_generation 0x6c // obsolete since Darwin v17 (High Sierra) -#define gtod_ns_base 0x70 // obsolete since Darwin v17 (High Sierra) -#define gtod_sec_base 0x78 // obsolete since Darwin v17 (High Sierra) - -#define v17_gtod_ns_base 0xd0 -#define v17_gtod_sec_ofs 0xd8 -#define v17_gtod_frac_ofs 0xe0 -#define v17_gtod_scale 0xe8 -#define v17_gtod_tkspersec 0xf0 - GLOBL timebase<>(SB),NOPTR,$(machTimebaseInfo__size) TEXT runtime·nanotime_trampoline(SB),NOSPLIT,$0 @@ -141,152 +123,13 @@ initialized: POPQ BP RET -TEXT time·now(SB), NOSPLIT, $32-24 - // Note: The 32 bytes of stack frame requested on the TEXT line - // are used in the systime fallback, as the timeval address - // filled in by the system call. - MOVQ $0x7fffffe00000, BP /* comm page base */ - CMPQ runtime·darwinVersion(SB), $17 - JB legacy /* sierra and older */ - - // This is the new code, for macOS High Sierra (Darwin v17) and newer. -v17: - // Loop trying to take a consistent snapshot - // of the time parameters. -timeloop17: - MOVQ v17_gtod_ns_base(BP), R12 - - MOVL nt_generation(BP), CX - TESTL CX, CX - JZ timeloop17 - RDTSC - MOVQ nt_tsc_base(BP), SI - MOVL nt_scale(BP), DI - MOVQ nt_ns_base(BP), BX - CMPL nt_generation(BP), CX - JNE timeloop17 - - MOVQ v17_gtod_sec_ofs(BP), R8 - MOVQ v17_gtod_frac_ofs(BP), R9 - MOVQ v17_gtod_scale(BP), R10 - MOVQ v17_gtod_tkspersec(BP), R11 - CMPQ v17_gtod_ns_base(BP), R12 - JNE timeloop17 - - // Compute monotonic time - // mono = ((tsc - nt_tsc_base) * nt_scale) >> 32 + nt_ns_base - // The multiply and shift extracts the top 64 bits of the 96-bit product. - SHLQ $32, DX - ADDQ DX, AX - SUBQ SI, AX - MULQ DI - SHRQ $32, AX:DX - ADDQ BX, AX - - // Subtract startNano base to return the monotonic runtime timer - // which is an offset from process boot. - MOVQ AX, BX - MOVQ runtime·startNano(SB), CX - SUBQ CX, BX - MOVQ BX, monotonic+16(FP) - - // Now compute the 128-bit wall time: - // wall = ((mono - gtod_ns_base) * gtod_scale) + gtod_offs - // The parameters are updated every second, so if we found them - // outdated (that is, more than one second is passed from the ns base), - // fallback to the syscall. - TESTQ R12, R12 - JZ systime - SUBQ R12, AX - CMPQ R11, AX - JB systime - MULQ R10 - ADDQ R9, AX - ADCQ R8, DX - - // Convert the 128-bit wall time into (sec,nsec). - // High part (seconds) is already good to go, while low part - // (fraction of seconds) must be converted to nanoseconds. - MOVQ DX, sec+0(FP) - MOVQ $1000000000, CX - MULQ CX - MOVQ DX, nsec+8(FP) - RET - - // This is the legacy code needed for macOS Sierra (Darwin v16) and older. -legacy: - // Loop trying to take a consistent snapshot - // of the time parameters. -timeloop: - MOVL gtod_generation(BP), R8 - MOVL nt_generation(BP), R9 - TESTL R9, R9 - JZ timeloop - RDTSC - MOVQ nt_tsc_base(BP), R10 - MOVL nt_scale(BP), R11 - MOVQ nt_ns_base(BP), R12 - CMPL nt_generation(BP), R9 - JNE timeloop - MOVQ gtod_ns_base(BP), R13 - MOVQ gtod_sec_base(BP), R14 - CMPL gtod_generation(BP), R8 - JNE timeloop - - // Gathered all the data we need. Compute: - // monotonic_time = ((tsc - nt_tsc_base) * nt_scale) >> 32 + nt_ns_base - // The multiply and shift extracts the top 64 bits of the 96-bit product. - SHLQ $32, DX - ADDQ DX, AX - SUBQ R10, AX - MULQ R11 - SHRQ $32, AX:DX - ADDQ R12, AX - MOVQ AX, BX - MOVQ runtime·startNano(SB), CX - SUBQ CX, BX - MOVQ BX, monotonic+16(FP) - - // Compute: - // wall_time = monotonic time - gtod_ns_base + gtod_sec_base*1e9 - // or, if gtod_generation==0, invoke the system call. - TESTL R8, R8 - JZ systime - SUBQ R13, AX - IMULQ $1000000000, R14 - ADDQ R14, AX - - // Split wall time into sec, nsec. - // generated code for - // func f(x uint64) (uint64, uint64) { return x/1e9, x%1e9 } - // adapted to reduce duplication - MOVQ AX, CX - SHRQ $9, AX - MOVQ $19342813113834067, DX - MULQ DX - SHRQ $11, DX - MOVQ DX, sec+0(FP) - IMULQ $1000000000, DX - SUBQ DX, CX - MOVL CX, nsec+8(FP) - RET - -systime: - // Fall back to system call (usually first call in this thread). - MOVQ SP, DI - MOVQ $0, SI - MOVQ $0, DX // required as of Sierra; Issue 16570 - MOVL $(0x2000000+116), AX // gettimeofday - SYSCALL - CMPQ AX, $0 - JNE inreg - MOVQ 0(SP), AX - MOVL 8(SP), DX -inreg: - // sec is in AX, usec in DX - IMULQ $1000, DX - MOVQ AX, sec+0(FP) - MOVL DX, nsec+8(FP) +TEXT runtime·walltime_trampoline(SB),NOSPLIT,$0 + PUSHQ BP // make a frame; keep stack aligned + MOVQ SP, BP + // DI already has *timeval + XORL SI, SI // no timezone needed + CALL libc_gettimeofday(SB) + POPQ BP RET TEXT runtime·sigprocmask(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_darwin_arm.s b/src/runtime/sys_darwin_arm.s index fcbcdbc42cafc..a940d95732c1e 100644 --- a/src/runtime/sys_darwin_arm.s +++ b/src/runtime/sys_darwin_arm.s @@ -124,24 +124,10 @@ TEXT runtime·setitimer(SB),NOSPLIT,$0 SWI $0x80 RET -TEXT runtime·walltime(SB), 7, $32 - MOVW $8(R13), R0 // timeval - MOVW $0, R1 // zone - MOVW $0, R2 // see issue 16570 - MOVW $SYS_gettimeofday, R12 - SWI $0x80 // Note: R0 is tv_sec, R1 is tv_usec - CMP $0, R0 - BNE inreg - MOVW 8(R13), R0 - MOVW 12(R13), R1 -inreg: - MOVW R1, R2 // usec - MOVW R0, sec_lo+0(FP) - MOVW $0, R1 - MOVW R1, sec_hi+4(FP) - MOVW $1000, R3 - MUL R3, R2 - MOVW R2, nsec+8(FP) +TEXT runtime·walltime_trampoline(SB),NOSPLIT,$0 + // R0 already has *timeval + MOVW $0, R1 // no timezone needed + BL libc_gettimeofday(SB) RET GLOBL timebase<>(SB),NOPTR,$(machTimebaseInfo__size) diff --git a/src/runtime/sys_darwin_arm64.s b/src/runtime/sys_darwin_arm64.s index f0d9032a60fc7..d13e44afcfa2f 100644 --- a/src/runtime/sys_darwin_arm64.s +++ b/src/runtime/sys_darwin_arm64.s @@ -116,22 +116,10 @@ TEXT runtime·setitimer(SB),NOSPLIT,$0 SVC $0x80 RET -TEXT runtime·walltime(SB),NOSPLIT,$40-12 - MOVD RSP, R0 // timeval - MOVD R0, R9 // this is how dyld calls gettimeofday - MOVW $0, R1 // zone - MOVD $0, R2 // see issue 16570 - MOVW $SYS_gettimeofday, R16 - SVC $0x80 // Note: x0 is tv_sec, w1 is tv_usec - CMP $0, R0 - BNE inreg - MOVD 0(RSP), R0 - MOVW 8(RSP), R1 -inreg: - MOVD R0, sec+0(FP) - MOVW $1000, R3 - MUL R3, R1 - MOVW R1, nsec+8(FP) +TEXT runtime·walltime_trampoline(SB),NOSPLIT,$0 + // R0 already has *timeval + MOVD $0, R1 // no timezone needed + BL libc_gettimeofday(SB) RET GLOBL timebase<>(SB),NOPTR,$(machTimebaseInfo__size) diff --git a/src/runtime/timeasm.go b/src/runtime/timeasm.go index 7474bec5565d2..5af920c18c048 100644 --- a/src/runtime/timeasm.go +++ b/src/runtime/timeasm.go @@ -6,7 +6,7 @@ // Those systems are also expected to have nanotime subtract startNano, // so that time.now and nanotime return the same monotonic clock readings. -// +build darwin,amd64 darwin,386 windows +// +build windows package runtime diff --git a/src/runtime/timestub.go b/src/runtime/timestub.go index a76a7619369ae..f9230da69f92a 100644 --- a/src/runtime/timestub.go +++ b/src/runtime/timestub.go @@ -5,7 +5,6 @@ // Declarations for operating systems implementing time.now // indirectly, in terms of walltime and nanotime assembly. -// +build !darwin !amd64,!386 // +build !windows package runtime diff --git a/src/runtime/timestub2.go b/src/runtime/timestub2.go index 8e15085d21b9b..9ddc6fed9182d 100644 --- a/src/runtime/timestub2.go +++ b/src/runtime/timestub2.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !darwin !amd64,!386 +// +build !darwin // +build !windows // +build !freebsd From 9e56156ade484d806cdd3aceb38f100b66d490bf Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 29 May 2018 22:08:32 +0000 Subject: [PATCH 027/187] net/http/httputil: pass through any "TE: trailers" header to backend Fixes #21096 Change-Id: I2a4688a79bdaa25b4e8ef38e3390d93d3d0bce04 Reviewed-on: https://go-review.googlesource.com/115135 Reviewed-by: Ian Lance Taylor --- src/net/http/httputil/reverseproxy.go | 15 +++++++++++++-- src/net/http/httputil/reverseproxy_test.go | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/net/http/httputil/reverseproxy.go b/src/net/http/httputil/reverseproxy.go index 80ee22895af6a..d5d0a505f7b93 100644 --- a/src/net/http/httputil/reverseproxy.go +++ b/src/net/http/httputil/reverseproxy.go @@ -178,9 +178,20 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // important is "Connection" because we want a persistent // connection, regardless of what the client sent to us. for _, h := range hopHeaders { - if outreq.Header.Get(h) != "" { - outreq.Header.Del(h) + hv := outreq.Header.Get(h) + if hv == "" { + continue } + if h == "Te" && hv == "trailers" { + // Issue 21096: tell backend applications that + // care about trailer support that we support + // trailers. (We do, but we don't go out of + // our way to advertise that unless the + // incoming client request thought it was + // worth mentioning) + continue + } + outreq.Header.Del(h) } if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil { diff --git a/src/net/http/httputil/reverseproxy_test.go b/src/net/http/httputil/reverseproxy_test.go index 3dcc5c72874fa..1ad67562af047 100644 --- a/src/net/http/httputil/reverseproxy_test.go +++ b/src/net/http/httputil/reverseproxy_test.go @@ -49,6 +49,9 @@ func TestReverseProxy(t *testing.T) { if c := r.Header.Get("Connection"); c != "" { t.Errorf("handler got Connection header value %q", c) } + if c := r.Header.Get("Te"); c != "trailers" { + t.Errorf("handler got Te header value %q; want 'trailers'", c) + } if c := r.Header.Get("Upgrade"); c != "" { t.Errorf("handler got Upgrade header value %q", c) } @@ -85,6 +88,7 @@ func TestReverseProxy(t *testing.T) { getReq, _ := http.NewRequest("GET", frontend.URL, nil) getReq.Host = "some-name" getReq.Header.Set("Connection", "close") + getReq.Header.Set("Te", "trailers") getReq.Header.Set("Proxy-Connection", "should be deleted") getReq.Header.Set("Upgrade", "foo") getReq.Close = true From a4330ed694c588d495f7c72a9cbb0cd39dde31e8 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 29 May 2018 20:05:12 +0000 Subject: [PATCH 028/187] net/http: document Server's implicit Content-Length response header Fixes #23450 Change-Id: I829399194299d2e6d5e754b60e8f72b321b5da90 Reviewed-on: https://go-review.googlesource.com/115040 Reviewed-by: Ian Lance Taylor --- src/net/http/server.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/net/http/server.go b/src/net/http/server.go index 2d3486dc66083..699cc0a1806d2 100644 --- a/src/net/http/server.go +++ b/src/net/http/server.go @@ -107,7 +107,7 @@ type ResponseWriter interface { // is to prefix the Header map keys with the TrailerPrefix // constant value. See TrailerPrefix. // - // To suppress implicit response headers (such as "Date"), set + // To suppress automatic response headers (such as "Date"), set // their value to nil. Header() Header @@ -117,7 +117,9 @@ type ResponseWriter interface { // WriteHeader(http.StatusOK) before writing the data. If the Header // does not contain a Content-Type line, Write adds a Content-Type set // to the result of passing the initial 512 bytes of written data to - // DetectContentType. + // DetectContentType. Additionally, if the total size of all written + // data is under a few KB and there are no Flush calls, the + // Content-Length header is added automatically. // // Depending on the HTTP protocol version and the client, calling // Write or WriteHeader may prevent future reads on the From c6295e72bf9536d75d3528ad17bdd49f2856b58d Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Tue, 29 May 2018 23:53:19 +0100 Subject: [PATCH 029/187] net: move dial and listen functions under sysDialer, sysListener Updates #9661 Change-Id: I237e7502cb9faad6dece1e25b1a503739c54d826 Reviewed-on: https://go-review.googlesource.com/115175 Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/net/dial.go | 53 +++++++++++++++++++++----------------- src/net/dial_test.go | 18 +++++++------ src/net/iprawsock.go | 12 +++++++-- src/net/iprawsock_plan9.go | 4 +-- src/net/iprawsock_posix.go | 15 +++++------ src/net/tcpsock.go | 6 +++-- src/net/tcpsock_plan9.go | 18 ++++++------- src/net/tcpsock_posix.go | 16 ++++++------ src/net/udpsock.go | 9 ++++--- src/net/udpsock_plan9.go | 12 ++++----- src/net/udpsock_posix.go | 12 ++++----- src/net/unixsock.go | 9 ++++--- src/net/unixsock_plan9.go | 6 ++--- src/net/unixsock_posix.go | 12 ++++----- 14 files changed, 111 insertions(+), 91 deletions(-) diff --git a/src/net/dial.go b/src/net/dial.go index f8b4aa227420c..3ea049ca460e7 100644 --- a/src/net/dial.go +++ b/src/net/dial.go @@ -306,8 +306,8 @@ func DialTimeout(network, address string, timeout time.Duration) (Conn, error) { return d.Dial(network, address) } -// dialParam contains a Dial's parameters and configuration. -type dialParam struct { +// sysDialer contains a Dial's parameters and configuration. +type sysDialer struct { Dialer network, address string } @@ -377,7 +377,7 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn return nil, &OpError{Op: "dial", Net: network, Source: nil, Addr: nil, Err: err} } - dp := &dialParam{ + sd := &sysDialer{ Dialer: *d, network: network, address: address, @@ -392,9 +392,9 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn var c Conn if len(fallbacks) > 0 { - c, err = dialParallel(ctx, dp, primaries, fallbacks) + c, err = sd.dialParallel(ctx, primaries, fallbacks) } else { - c, err = dialSerial(ctx, dp, primaries) + c, err = sd.dialSerial(ctx, primaries) } if err != nil { return nil, err @@ -412,9 +412,9 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn // head start. It returns the first established connection and // closes the others. Otherwise it returns an error from the first // primary address. -func dialParallel(ctx context.Context, dp *dialParam, primaries, fallbacks addrList) (Conn, error) { +func (sd *sysDialer) dialParallel(ctx context.Context, primaries, fallbacks addrList) (Conn, error) { if len(fallbacks) == 0 { - return dialSerial(ctx, dp, primaries) + return sd.dialSerial(ctx, primaries) } returned := make(chan struct{}) @@ -433,7 +433,7 @@ func dialParallel(ctx context.Context, dp *dialParam, primaries, fallbacks addrL if !primary { ras = fallbacks } - c, err := dialSerial(ctx, dp, ras) + c, err := sd.dialSerial(ctx, ras) select { case results <- dialResult{Conn: c, error: err, primary: primary, done: true}: case <-returned: @@ -451,7 +451,7 @@ func dialParallel(ctx context.Context, dp *dialParam, primaries, fallbacks addrL go startRacer(primaryCtx, true) // Start the timer for the fallback racer. - fallbackTimer := time.NewTimer(dp.fallbackDelay()) + fallbackTimer := time.NewTimer(sd.fallbackDelay()) defer fallbackTimer.Stop() for { @@ -486,13 +486,13 @@ func dialParallel(ctx context.Context, dp *dialParam, primaries, fallbacks addrL // dialSerial connects to a list of addresses in sequence, returning // either the first successful connection, or the first error. -func dialSerial(ctx context.Context, dp *dialParam, ras addrList) (Conn, error) { +func (sd *sysDialer) dialSerial(ctx context.Context, ras addrList) (Conn, error) { var firstErr error // The error from the first address is most relevant. for i, ra := range ras { select { case <-ctx.Done(): - return nil, &OpError{Op: "dial", Net: dp.network, Source: dp.LocalAddr, Addr: ra, Err: mapErr(ctx.Err())} + return nil, &OpError{Op: "dial", Net: sd.network, Source: sd.LocalAddr, Addr: ra, Err: mapErr(ctx.Err())} default: } @@ -501,7 +501,7 @@ func dialSerial(ctx context.Context, dp *dialParam, ras addrList) (Conn, error) if err != nil { // Ran out of time. if firstErr == nil { - firstErr = &OpError{Op: "dial", Net: dp.network, Source: dp.LocalAddr, Addr: ra, Err: err} + firstErr = &OpError{Op: "dial", Net: sd.network, Source: sd.LocalAddr, Addr: ra, Err: err} } break } @@ -512,7 +512,7 @@ func dialSerial(ctx context.Context, dp *dialParam, ras addrList) (Conn, error) defer cancel() } - c, err := dialSingle(dialCtx, dp, ra) + c, err := sd.dialSingle(dialCtx, ra) if err == nil { return c, nil } @@ -522,47 +522,52 @@ func dialSerial(ctx context.Context, dp *dialParam, ras addrList) (Conn, error) } if firstErr == nil { - firstErr = &OpError{Op: "dial", Net: dp.network, Source: nil, Addr: nil, Err: errMissingAddress} + firstErr = &OpError{Op: "dial", Net: sd.network, Source: nil, Addr: nil, Err: errMissingAddress} } return nil, firstErr } // dialSingle attempts to establish and returns a single connection to // the destination address. -func dialSingle(ctx context.Context, dp *dialParam, ra Addr) (c Conn, err error) { +func (sd *sysDialer) dialSingle(ctx context.Context, ra Addr) (c Conn, err error) { trace, _ := ctx.Value(nettrace.TraceKey{}).(*nettrace.Trace) if trace != nil { raStr := ra.String() if trace.ConnectStart != nil { - trace.ConnectStart(dp.network, raStr) + trace.ConnectStart(sd.network, raStr) } if trace.ConnectDone != nil { - defer func() { trace.ConnectDone(dp.network, raStr, err) }() + defer func() { trace.ConnectDone(sd.network, raStr, err) }() } } - la := dp.LocalAddr + la := sd.LocalAddr switch ra := ra.(type) { case *TCPAddr: la, _ := la.(*TCPAddr) - c, err = dialTCP(ctx, dp.network, la, ra) + c, err = sd.dialTCP(ctx, la, ra) case *UDPAddr: la, _ := la.(*UDPAddr) - c, err = dialUDP(ctx, dp.network, la, ra) + c, err = sd.dialUDP(ctx, la, ra) case *IPAddr: la, _ := la.(*IPAddr) - c, err = dialIP(ctx, dp.network, la, ra) + c, err = sd.dialIP(ctx, la, ra) case *UnixAddr: la, _ := la.(*UnixAddr) - c, err = dialUnix(ctx, dp.network, la, ra) + c, err = sd.dialUnix(ctx, la, ra) default: - return nil, &OpError{Op: "dial", Net: dp.network, Source: la, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: dp.address}} + return nil, &OpError{Op: "dial", Net: sd.network, Source: la, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: sd.address}} } if err != nil { - return nil, &OpError{Op: "dial", Net: dp.network, Source: la, Addr: ra, Err: err} // c is non-nil interface containing nil pointer + return nil, &OpError{Op: "dial", Net: sd.network, Source: la, Addr: ra, Err: err} // c is non-nil interface containing nil pointer } return c, nil } +// sysListener contains a Listen's parameters and configuration. +type sysListener struct { + network, address string +} + // Listen announces on the local network address. // // The network must be "tcp", "tcp4", "tcp6", "unix" or "unixpacket". diff --git a/src/net/dial_test.go b/src/net/dial_test.go index 811e417cd7c1d..96d8921ec8525 100644 --- a/src/net/dial_test.go +++ b/src/net/dial_test.go @@ -142,8 +142,9 @@ const ( // In some environments, the slow IPs may be explicitly unreachable, and fail // more quickly than expected. This test hook prevents dialTCP from returning // before the deadline. -func slowDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - c, err := doDialTCP(ctx, net, laddr, raddr) +func slowDialTCP(ctx context.Context, network string, laddr, raddr *TCPAddr) (*TCPConn, error) { + sd := &sysDialer{network: network, address: raddr.String()} + c, err := sd.doDialTCP(ctx, laddr, raddr) if ParseIP(slowDst4).Equal(raddr.IP) || ParseIP(slowDst6).Equal(raddr.IP) { // Wait for the deadline, or indefinitely if none exists. <-ctx.Done() @@ -295,12 +296,12 @@ func TestDialParallel(t *testing.T) { FallbackDelay: fallbackDelay, } startTime := time.Now() - dp := &dialParam{ + sd := &sysDialer{ Dialer: d, network: "tcp", address: "?", } - c, err := dialParallel(context.Background(), dp, primaries, fallbacks) + c, err := sd.dialParallel(context.Background(), primaries, fallbacks) elapsed := time.Since(startTime) if c != nil { @@ -331,7 +332,7 @@ func TestDialParallel(t *testing.T) { wg.Done() }() startTime = time.Now() - c, err = dialParallel(ctx, dp, primaries, fallbacks) + c, err = sd.dialParallel(ctx, primaries, fallbacks) if c != nil { c.Close() } @@ -467,13 +468,14 @@ func TestDialParallelSpuriousConnection(t *testing.T) { // Now ignore the provided context (which will be canceled) and use a // different one to make sure this completes with a valid connection, // which we hope to be closed below: - return doDialTCP(context.Background(), net, laddr, raddr) + sd := &sysDialer{network: net, address: raddr.String()} + return sd.doDialTCP(context.Background(), laddr, raddr) } d := Dialer{ FallbackDelay: fallbackDelay, } - dp := &dialParam{ + sd := &sysDialer{ Dialer: d, network: "tcp", address: "?", @@ -488,7 +490,7 @@ func TestDialParallelSpuriousConnection(t *testing.T) { } // dialParallel returns one connection (and closes the other.) - c, err := dialParallel(context.Background(), dp, makeAddr("127.0.0.1"), makeAddr("::1")) + c, err := sd.dialParallel(context.Background(), makeAddr("127.0.0.1"), makeAddr("::1")) if err != nil { t.Fatal(err) } diff --git a/src/net/iprawsock.go b/src/net/iprawsock.go index 72cbc3943376f..95bab50e4ec97 100644 --- a/src/net/iprawsock.go +++ b/src/net/iprawsock.go @@ -209,7 +209,11 @@ func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } // If the IP field of raddr is nil or an unspecified IP address, the // local system is assumed. func DialIP(network string, laddr, raddr *IPAddr) (*IPConn, error) { - c, err := dialIP(context.Background(), network, laddr, raddr) + if raddr == nil { + return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + } + sd := &sysDialer{network: network, address: raddr.String()} + c, err := sd.dialIP(context.Background(), laddr, raddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } @@ -224,7 +228,11 @@ func DialIP(network string, laddr, raddr *IPAddr) (*IPConn, error) { // ListenIP listens on all available IP addresses of the local system // except multicast IP addresses. func ListenIP(network string, laddr *IPAddr) (*IPConn, error) { - c, err := listenIP(context.Background(), network, laddr) + if laddr == nil { + laddr = &IPAddr{} + } + sl := &sysListener{network: network, address: laddr.String()} + c, err := sl.listenIP(context.Background(), laddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } diff --git a/src/net/iprawsock_plan9.go b/src/net/iprawsock_plan9.go index 6aebea169ca51..ebe58088642de 100644 --- a/src/net/iprawsock_plan9.go +++ b/src/net/iprawsock_plan9.go @@ -25,10 +25,10 @@ func (c *IPConn) writeMsg(b, oob []byte, addr *IPAddr) (n, oobn int, err error) return 0, 0, syscall.EPLAN9 } -func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn, error) { +func (sd *sysDialer) dialIP(ctx context.Context, laddr, raddr *IPAddr) (*IPConn, error) { return nil, syscall.EPLAN9 } -func listenIP(ctx context.Context, netProto string, laddr *IPAddr) (*IPConn, error) { +func (sl *sysListener) listenIP(ctx context.Context, laddr *IPAddr) (*IPConn, error) { return nil, syscall.EPLAN9 } diff --git a/src/net/iprawsock_posix.go b/src/net/iprawsock_posix.go index 64c601602fe88..7dafd20bf6861 100644 --- a/src/net/iprawsock_posix.go +++ b/src/net/iprawsock_posix.go @@ -112,18 +112,15 @@ func (c *IPConn) writeMsg(b, oob []byte, addr *IPAddr) (n, oobn int, err error) return c.fd.writeMsg(b, oob, sa) } -func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn, error) { - network, proto, err := parseNetwork(ctx, netProto, true) +func (sd *sysDialer) dialIP(ctx context.Context, laddr, raddr *IPAddr) (*IPConn, error) { + network, proto, err := parseNetwork(ctx, sd.network, true) if err != nil { return nil, err } switch network { case "ip", "ip4", "ip6": default: - return nil, UnknownNetworkError(netProto) - } - if raddr == nil { - return nil, errMissingAddress + return nil, UnknownNetworkError(sd.network) } fd, err := internetSocket(ctx, network, laddr, raddr, syscall.SOCK_RAW, proto, "dial") if err != nil { @@ -132,15 +129,15 @@ func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn return newIPConn(fd), nil } -func listenIP(ctx context.Context, netProto string, laddr *IPAddr) (*IPConn, error) { - network, proto, err := parseNetwork(ctx, netProto, true) +func (sl *sysListener) listenIP(ctx context.Context, laddr *IPAddr) (*IPConn, error) { + network, proto, err := parseNetwork(ctx, sl.network, true) if err != nil { return nil, err } switch network { case "ip", "ip4", "ip6": default: - return nil, UnknownNetworkError(netProto) + return nil, UnknownNetworkError(sl.network) } fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_RAW, proto, "listen") if err != nil { diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index 0421ce56749d5..6e628f667f502 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -212,7 +212,8 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { if raddr == nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } - c, err := dialTCP(context.Background(), network, laddr, raddr) + sd := &sysDialer{network: network, address: raddr.String()} + c, err := sd.dialTCP(context.Background(), laddr, raddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } @@ -328,7 +329,8 @@ func ListenTCP(network string, laddr *TCPAddr) (*TCPListener, error) { if laddr == nil { laddr = &TCPAddr{} } - ln, err := listenTCP(context.Background(), network, laddr) + sl := &sysListener{network: network, address: laddr.String()} + ln, err := sl.listenTCP(context.Background(), laddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } diff --git a/src/net/tcpsock_plan9.go b/src/net/tcpsock_plan9.go index e37f0657c0798..f70ef6f43abb9 100644 --- a/src/net/tcpsock_plan9.go +++ b/src/net/tcpsock_plan9.go @@ -14,23 +14,23 @@ func (c *TCPConn) readFrom(r io.Reader) (int64, error) { return genericReadFrom(c, r) } -func dialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { +func (sd *sysDialer) dialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConn, error) { if testHookDialTCP != nil { - return testHookDialTCP(ctx, net, laddr, raddr) + return testHookDialTCP(ctx, sd.network, laddr, raddr) } - return doDialTCP(ctx, net, laddr, raddr) + return sd.doDialTCP(ctx, laddr, raddr) } -func doDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - switch net { +func (sd *sysDialer) doDialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConn, error) { + switch sd.network { case "tcp", "tcp4", "tcp6": default: - return nil, UnknownNetworkError(net) + return nil, UnknownNetworkError(sd.network) } if raddr == nil { return nil, errMissingAddress } - fd, err := dialPlan9(ctx, net, laddr, raddr) + fd, err := dialPlan9(ctx, sd.network, laddr, raddr) if err != nil { return nil, err } @@ -69,8 +69,8 @@ func (ln *TCPListener) file() (*os.File, error) { return f, nil } -func listenTCP(ctx context.Context, network string, laddr *TCPAddr) (*TCPListener, error) { - fd, err := listenPlan9(ctx, network, laddr) +func (sl *sysListener) listenTCP(ctx context.Context, laddr *TCPAddr) (*TCPListener, error) { + fd, err := listenPlan9(ctx, sl.network, laddr) if err != nil { return nil, err } diff --git a/src/net/tcpsock_posix.go b/src/net/tcpsock_posix.go index f6fd93158ac42..6061c16986c22 100644 --- a/src/net/tcpsock_posix.go +++ b/src/net/tcpsock_posix.go @@ -54,15 +54,15 @@ func (c *TCPConn) readFrom(r io.Reader) (int64, error) { return genericReadFrom(c, r) } -func dialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { +func (sd *sysDialer) dialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConn, error) { if testHookDialTCP != nil { - return testHookDialTCP(ctx, net, laddr, raddr) + return testHookDialTCP(ctx, sd.network, laddr, raddr) } - return doDialTCP(ctx, net, laddr, raddr) + return sd.doDialTCP(ctx, laddr, raddr) } -func doDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - fd, err := internetSocket(ctx, net, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") +func (sd *sysDialer) doDialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConn, error) { + fd, err := internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") // TCP has a rarely used mechanism called a 'simultaneous connection' in // which Dial("tcp", addr1, addr2) run on the machine at addr1 can @@ -92,7 +92,7 @@ func doDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn if err == nil { fd.Close() } - fd, err = internetSocket(ctx, net, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") + fd, err = internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") } if err != nil { @@ -155,8 +155,8 @@ func (ln *TCPListener) file() (*os.File, error) { return f, nil } -func listenTCP(ctx context.Context, network string, laddr *TCPAddr) (*TCPListener, error) { - fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_STREAM, 0, "listen") +func (sl *sysListener) listenTCP(ctx context.Context, laddr *TCPAddr) (*TCPListener, error) { + fd, err := internetSocket(ctx, sl.network, laddr, nil, syscall.SOCK_STREAM, 0, "listen") if err != nil { return nil, err } diff --git a/src/net/udpsock.go b/src/net/udpsock.go index 158265f06f850..937b9270bdd29 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -208,7 +208,8 @@ func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { if raddr == nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } - c, err := dialUDP(context.Background(), network, laddr, raddr) + sd := &sysDialer{network: network, address: raddr.String()} + c, err := sd.dialUDP(context.Background(), laddr, raddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } @@ -233,7 +234,8 @@ func ListenUDP(network string, laddr *UDPAddr) (*UDPConn, error) { if laddr == nil { laddr = &UDPAddr{} } - c, err := listenUDP(context.Background(), network, laddr) + sl := &sysListener{network: network, address: laddr.String()} + c, err := sl.listenUDP(context.Background(), laddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } @@ -266,7 +268,8 @@ func ListenMulticastUDP(network string, ifi *Interface, gaddr *UDPAddr) (*UDPCon if gaddr == nil || gaddr.IP == nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: errMissingAddress} } - c, err := listenMulticastUDP(context.Background(), network, ifi, gaddr) + sl := &sysListener{network: network, address: gaddr.String()} + c, err := sl.listenMulticastUDP(context.Background(), ifi, gaddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: err} } diff --git a/src/net/udpsock_plan9.go b/src/net/udpsock_plan9.go index 1ce7f88c62c29..563d943507e44 100644 --- a/src/net/udpsock_plan9.go +++ b/src/net/udpsock_plan9.go @@ -55,8 +55,8 @@ func (c *UDPConn) writeMsg(b, oob []byte, addr *UDPAddr) (n, oobn int, err error return 0, 0, syscall.EPLAN9 } -func dialUDP(ctx context.Context, net string, laddr, raddr *UDPAddr) (*UDPConn, error) { - fd, err := dialPlan9(ctx, net, laddr, raddr) +func (sd *sysDialer) dialUDP(ctx context.Context, laddr, raddr *UDPAddr) (*UDPConn, error) { + fd, err := dialPlan9(ctx, sd.network, laddr, raddr) if err != nil { return nil, err } @@ -91,8 +91,8 @@ func unmarshalUDPHeader(b []byte) (*udpHeader, []byte) { return h, b } -func listenUDP(ctx context.Context, network string, laddr *UDPAddr) (*UDPConn, error) { - l, err := listenPlan9(ctx, network, laddr) +func (sl *sysListener) listenUDP(ctx context.Context, laddr *UDPAddr) (*UDPConn, error) { + l, err := listenPlan9(ctx, sl.network, laddr) if err != nil { return nil, err } @@ -108,8 +108,8 @@ func listenUDP(ctx context.Context, network string, laddr *UDPAddr) (*UDPConn, e return newUDPConn(fd), err } -func listenMulticastUDP(ctx context.Context, network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - l, err := listenPlan9(ctx, network, gaddr) +func (sl *sysListener) listenMulticastUDP(ctx context.Context, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + l, err := listenPlan9(ctx, sl.network, gaddr) if err != nil { return nil, err } diff --git a/src/net/udpsock_posix.go b/src/net/udpsock_posix.go index a126506d3423c..4e96f4781df47 100644 --- a/src/net/udpsock_posix.go +++ b/src/net/udpsock_posix.go @@ -94,24 +94,24 @@ func (c *UDPConn) writeMsg(b, oob []byte, addr *UDPAddr) (n, oobn int, err error return c.fd.writeMsg(b, oob, sa) } -func dialUDP(ctx context.Context, net string, laddr, raddr *UDPAddr) (*UDPConn, error) { - fd, err := internetSocket(ctx, net, laddr, raddr, syscall.SOCK_DGRAM, 0, "dial") +func (sd *sysDialer) dialUDP(ctx context.Context, laddr, raddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_DGRAM, 0, "dial") if err != nil { return nil, err } return newUDPConn(fd), nil } -func listenUDP(ctx context.Context, network string, laddr *UDPAddr) (*UDPConn, error) { - fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_DGRAM, 0, "listen") +func (sl *sysListener) listenUDP(ctx context.Context, laddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, sl.network, laddr, nil, syscall.SOCK_DGRAM, 0, "listen") if err != nil { return nil, err } return newUDPConn(fd), nil } -func listenMulticastUDP(ctx context.Context, network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - fd, err := internetSocket(ctx, network, gaddr, nil, syscall.SOCK_DGRAM, 0, "listen") +func (sl *sysListener) listenMulticastUDP(ctx context.Context, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, sl.network, gaddr, nil, syscall.SOCK_DGRAM, 0, "listen") if err != nil { return nil, err } diff --git a/src/net/unixsock.go b/src/net/unixsock.go index 551280f93622e..bd7dc398487f9 100644 --- a/src/net/unixsock.go +++ b/src/net/unixsock.go @@ -200,7 +200,8 @@ func DialUnix(network string, laddr, raddr *UnixAddr) (*UnixConn, error) { default: return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(network)} } - c, err := dialUnix(context.Background(), network, laddr, raddr) + sd := &sysDialer{network: network, address: raddr.String()} + c, err := sd.dialUnix(context.Background(), laddr, raddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } @@ -316,7 +317,8 @@ func ListenUnix(network string, laddr *UnixAddr) (*UnixListener, error) { if laddr == nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} } - ln, err := listenUnix(context.Background(), network, laddr) + sa := &sysListener{network: network, address: laddr.String()} + ln, err := sa.listenUnix(context.Background(), laddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } @@ -335,7 +337,8 @@ func ListenUnixgram(network string, laddr *UnixAddr) (*UnixConn, error) { if laddr == nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: errMissingAddress} } - c, err := listenUnixgram(context.Background(), network, laddr) + sa := &sysListener{network: network, address: laddr.String()} + c, err := sa.listenUnixgram(context.Background(), laddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } diff --git a/src/net/unixsock_plan9.go b/src/net/unixsock_plan9.go index e70eb211bbf0b..6ebd4d7d3b46c 100644 --- a/src/net/unixsock_plan9.go +++ b/src/net/unixsock_plan9.go @@ -26,7 +26,7 @@ func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err err return 0, 0, syscall.EPLAN9 } -func dialUnix(ctx context.Context, network string, laddr, raddr *UnixAddr) (*UnixConn, error) { +func (sd *sysDialer) dialUnix(ctx context.Context, laddr, raddr *UnixAddr) (*UnixConn, error) { return nil, syscall.EPLAN9 } @@ -42,10 +42,10 @@ func (ln *UnixListener) file() (*os.File, error) { return nil, syscall.EPLAN9 } -func listenUnix(ctx context.Context, network string, laddr *UnixAddr) (*UnixListener, error) { +func (sl *sysListener) listenUnix(ctx context.Context, laddr *UnixAddr) (*UnixListener, error) { return nil, syscall.EPLAN9 } -func listenUnixgram(ctx context.Context, network string, laddr *UnixAddr) (*UnixConn, error) { +func (sl *sysListener) listenUnixgram(ctx context.Context, laddr *UnixAddr) (*UnixConn, error) { return nil, syscall.EPLAN9 } diff --git a/src/net/unixsock_posix.go b/src/net/unixsock_posix.go index a8f892e6c4659..f627567af5f0f 100644 --- a/src/net/unixsock_posix.go +++ b/src/net/unixsock_posix.go @@ -150,8 +150,8 @@ func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err err return c.fd.writeMsg(b, oob, sa) } -func dialUnix(ctx context.Context, net string, laddr, raddr *UnixAddr) (*UnixConn, error) { - fd, err := unixSocket(ctx, net, laddr, raddr, "dial") +func (sd *sysDialer) dialUnix(ctx context.Context, laddr, raddr *UnixAddr) (*UnixConn, error) { + fd, err := unixSocket(ctx, sd.network, laddr, raddr, "dial") if err != nil { return nil, err } @@ -206,16 +206,16 @@ func (l *UnixListener) SetUnlinkOnClose(unlink bool) { l.unlink = unlink } -func listenUnix(ctx context.Context, network string, laddr *UnixAddr) (*UnixListener, error) { - fd, err := unixSocket(ctx, network, laddr, nil, "listen") +func (sl *sysListener) listenUnix(ctx context.Context, laddr *UnixAddr) (*UnixListener, error) { + fd, err := unixSocket(ctx, sl.network, laddr, nil, "listen") if err != nil { return nil, err } return &UnixListener{fd: fd, path: fd.laddr.String(), unlink: true}, nil } -func listenUnixgram(ctx context.Context, network string, laddr *UnixAddr) (*UnixConn, error) { - fd, err := unixSocket(ctx, network, laddr, nil, "listen") +func (sl *sysListener) listenUnixgram(ctx context.Context, laddr *UnixAddr) (*UnixConn, error) { + fd, err := unixSocket(ctx, sl.network, laddr, nil, "listen") if err != nil { return nil, err } From 7d704a97608292150263f34da9430ac70a8c4ffd Mon Sep 17 00:00:00 2001 From: Mikio Hara Date: Wed, 30 May 2018 12:52:49 +0900 Subject: [PATCH 030/187] net: fix ExampleUDPConn_WriteTo Change-Id: I174b17395509d4c9fb55332c2405890b2a350cbd Reviewed-on: https://go-review.googlesource.com/115218 Run-TryBot: Mikio Hara TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/net/example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/example_test.go b/src/net/example_test.go index c6eb75d0a4e68..8126a28404a41 100644 --- a/src/net/example_test.go +++ b/src/net/example_test.go @@ -128,7 +128,7 @@ func ExampleUDPConn_WriteTo() { } defer conn.Close() - dst, err := net.ResolveIPAddr("udp", "192.0.2.1:2000") + dst, err := net.ResolveUDPAddr("udp", "192.0.2.1:2000") if err != nil { log.Fatal(err) } From e023d5b0ac37310ce8a06e33e751a9a7c632046d Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 30 May 2018 10:15:56 +0200 Subject: [PATCH 031/187] net/http: fix typo in comment Change-Id: Ibb21c12bf67b2648eb7606bee8ec1b54e6c70dd5 Reviewed-on: https://go-review.googlesource.com/115237 Reviewed-by: Alberto Donizetti --- src/net/http/socks_bundle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/http/socks_bundle.go b/src/net/http/socks_bundle.go index 5c3830a9e6dec..8b347898e8501 100644 --- a/src/net/http/socks_bundle.go +++ b/src/net/http/socks_bundle.go @@ -231,7 +231,7 @@ const ( socksAuthMethodNotRequired socksAuthMethod = 0x00 // no authentication required socksAuthMethodUsernamePassword socksAuthMethod = 0x02 // use username/password - socksAuthMethodNoAcceptableMethods socksAuthMethod = 0xff // no acceptable authetication methods + socksAuthMethodNoAcceptableMethods socksAuthMethod = 0xff // no acceptable authentication methods socksStatusSucceeded socksReply = 0x00 ) From 679004f484f0902b6eb8c66eb40c88f98a1163f1 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 30 May 2018 10:10:08 +0200 Subject: [PATCH 032/187] cmd/dist: remove support for macOS 10.9 and earlier Updates #23122 Change-Id: I14cfb83f3f78cdbe5880bd29209388ad12b9ee89 Reviewed-on: https://go-review.googlesource.com/115236 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor Reviewed-by: Brad Fitzpatrick --- src/cmd/dist/main.go | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/src/cmd/dist/main.go b/src/cmd/dist/main.go index a72a2607f9f59..37e37e2733566 100644 --- a/src/cmd/dist/main.go +++ b/src/cmd/dist/main.go @@ -9,7 +9,6 @@ import ( "fmt" "os" "runtime" - "strconv" "strings" ) @@ -61,6 +60,8 @@ func main() { // Even on 64-bit platform, darwin uname -m prints i386. // We don't support any of the OS X versions that run on 32-bit-only hardware anymore. gohostarch = "amd64" + // macOS 10.9 and later require clang + defaultclang = true case "freebsd": // Since FreeBSD 10 gcc is no longer part of the base system. defaultclang = true @@ -126,29 +127,6 @@ func main() { } bginit() - // The OS X 10.6 linker does not support external linking mode. - // See golang.org/issue/5130. - // - // OS X 10.6 does not work with clang either, but OS X 10.9 requires it. - // It seems to work with OS X 10.8, so we default to clang for 10.8 and later. - // See golang.org/issue/5822. - // - // Roughly, OS X 10.N shows up as uname release (N+4), - // so OS X 10.6 is uname version 10 and OS X 10.8 is uname version 12. - if gohostos == "darwin" { - rel := run("", CheckExit, "uname", "-r") - if i := strings.Index(rel, "."); i >= 0 { - rel = rel[:i] - } - osx, _ := strconv.Atoi(rel) - if osx <= 6+4 { - goextlinkenabled = "0" - } - if osx >= 8+4 { - defaultclang = true - } - } - if len(os.Args) > 1 && os.Args[1] == "-check-goarm" { useVFPv1() // might fail with SIGILL println("VFPv1 OK.") From f5cf72d43ed1441d2e80654be5bde4b11f4cbcd1 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 30 May 2018 09:52:06 +0200 Subject: [PATCH 033/187] cmd/link/internal/ld: drop duplicate copyright comment The copyright message already appears at the top of macho.go. Drop the duplicate further down in the file. Change-Id: Ib0a69f568c4ef656bab14176223936cd2fe078d1 Reviewed-on: https://go-review.googlesource.com/115235 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/link/internal/ld/macho.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/cmd/link/internal/ld/macho.go b/src/cmd/link/internal/ld/macho.go index 8643fef043cae..c0083fb8b0d3e 100644 --- a/src/cmd/link/internal/ld/macho.go +++ b/src/cmd/link/internal/ld/macho.go @@ -142,10 +142,6 @@ const ( S_ATTR_SOME_INSTRUCTIONS = 0x00000400 ) -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - // Mach-O file writing // http://developer.apple.com/mac/library/DOCUMENTATION/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html From fd4392ba6f8c593fdfdf19366f3896b668db2824 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Tue, 29 May 2018 15:13:32 -0700 Subject: [PATCH 034/187] go/types: don't over-eagerly verify embedded interfaces In https://go-review.googlesource.com/c/go/+/114317 (fix for #25301) the constructor types.NewInterface was replaced with NewInterface2. The new constructor aggressively verified that embedded interfaces had an underlying type of interface type; the old code didn't do any verification. During importing, defined types may be not yet fully set up, and testing their underlying types will fail in those cases. This change only verifies embedded types that are not defined types and thus restores behavior for defined types to how it was before the fix for #25301. Fixes #25596. Fixes #25615. Change-Id: Ifd694413656ec0b780fe4f37acaa9e6ba6077271 Reviewed-on: https://go-review.googlesource.com/115155 Run-TryBot: Robert Griesemer TryBot-Result: Gobot Gobot Reviewed-by: Alan Donovan --- src/go/internal/gcimporter/gcimporter_test.go | 21 +++++++++++++++++++ .../gcimporter/testdata/issue25596.go | 13 ++++++++++++ src/go/types/type.go | 10 +++++++-- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 src/go/internal/gcimporter/testdata/issue25596.go diff --git a/src/go/internal/gcimporter/gcimporter_test.go b/src/go/internal/gcimporter/gcimporter_test.go index a8745eea3e385..308f93e8bd786 100644 --- a/src/go/internal/gcimporter/gcimporter_test.go +++ b/src/go/internal/gcimporter/gcimporter_test.go @@ -530,6 +530,27 @@ func TestIssue25301(t *testing.T) { importPkg(t, "./testdata/issue25301") } +func TestIssue25596(t *testing.T) { + skipSpecialPlatforms(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + if f := compile(t, "testdata", "issue25596.go"); f != "" { + defer os.Remove(f) + } + + importPkg(t, "./testdata/issue25596") +} + func importPkg(t *testing.T, path string) *types.Package { pkg, err := Import(make(map[string]*types.Package), path, ".", nil) if err != nil { diff --git a/src/go/internal/gcimporter/testdata/issue25596.go b/src/go/internal/gcimporter/testdata/issue25596.go new file mode 100644 index 0000000000000..8923373e5fa44 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/issue25596.go @@ -0,0 +1,13 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue25596 + +type E interface { + M() T +} + +type T interface { + E +} diff --git a/src/go/types/type.go b/src/go/types/type.go index f274e30ab6b9b..cc87f1edb57d9 100644 --- a/src/go/types/type.go +++ b/src/go/types/type.go @@ -275,7 +275,9 @@ func NewInterface(methods []*Func, embeddeds []*Named) *Interface { } // NewInterface2 returns a new (incomplete) interface for the given methods and embedded types. -// Each embedded type must have an underlying type of interface type. +// Each embedded type must have an underlying type of interface type (this property is not +// verified for defined types, which may be in the process of being set up and which don't +// have a valid underlying type yet). // NewInterface2 takes ownership of the provided methods and may modify their types by setting // missing receivers. To compute the method set of the interface, Complete must be called. func NewInterface2(methods []*Func, embeddeds []Type) *Interface { @@ -298,8 +300,12 @@ func NewInterface2(methods []*Func, embeddeds []Type) *Interface { sort.Sort(byUniqueMethodName(methods)) if len(embeddeds) > 0 { + // All embedded types should be interfaces; however, defined types + // may not yet be fully resolved. Only verify that non-defined types + // are interfaces. This matches the behavior of the code before the + // fix for #25301 (issue #25596). for _, t := range embeddeds { - if !IsInterface(t) { + if _, ok := t.(*Named); !ok && !IsInterface(t) { panic("embedded type is not an interface") } } From 85f4051731f9f2d0514301470d528db94ed5781c Mon Sep 17 00:00:00 2001 From: Brian Kessler Date: Wed, 6 Dec 2017 09:53:14 -0700 Subject: [PATCH 035/187] math/big: implement Atkin's ModSqrt for 5 mod 8 primes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For primes congruent to 5 mod 8 there is a simple deterministic method for calculating the modular square root due to Atkin, using one exponentiation and 4 multiplications. A. Atkin. Probabilistic primality testing, summary by F. Morain. Research Report 1779, INRIA, pages 159–163, 1992. This increases the speed of modular square roots for these primes considerably. name old time/op new time/op delta ModSqrt231_5Mod8-4 1.03ms ± 2% 0.36ms ± 5% -65.06% (p=0.008 n=5+5) Change-Id: I024f6e514bbca8d634218983117db2afffe615fe Reviewed-on: https://go-review.googlesource.com/99615 Reviewed-by: Robert Griesemer --- src/math/big/int.go | 37 +++++++++++++++++++++++++++++++++---- src/math/big/int_test.go | 30 ++++++++++++++---------------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/src/math/big/int.go b/src/math/big/int.go index b1d09cdad803b..d46b5d8a86121 100644 --- a/src/math/big/int.go +++ b/src/math/big/int.go @@ -819,6 +819,30 @@ func (z *Int) modSqrt3Mod4Prime(x, p *Int) *Int { return z } +// modSqrt5Mod8 uses Atkin's observation that 2 is not a square mod p +// alpha == (2*a)^((p-5)/8) mod p +// beta == 2*a*alpha^2 mod p is a square root of -1 +// b == a*alpha*(beta-1) mod p is a square root of a +// to calculate the square root of any quadratic residue mod p quickly for 5 +// mod 8 primes. +func (z *Int) modSqrt5Mod8Prime(x, p *Int) *Int { + // p == 5 mod 8 implies p = e*8 + 5 + // e is the quotient and 5 the remainder on division by 8 + e := new(Int).Rsh(p, 3) // e = (p - 5) / 8 + tx := new(Int).Lsh(x, 1) // tx = 2*x + alpha := new(Int).Exp(tx, e, p) + beta := new(Int).Mul(alpha, alpha) + beta.Mod(beta, p) + beta.Mul(beta, tx) + beta.Mod(beta, p) + beta.Sub(beta, intOne) + beta.Mul(beta, x) + beta.Mod(beta, p) + beta.Mul(beta, alpha) + z.Mod(beta, p) + return z +} + // modSqrtTonelliShanks uses the Tonelli-Shanks algorithm to find the square // root of a quadratic residue modulo any prime. func (z *Int) modSqrtTonelliShanks(x, p *Int) *Int { @@ -885,12 +909,17 @@ func (z *Int) ModSqrt(x, p *Int) *Int { x = new(Int).Mod(x, p) } - // Check whether p is 3 mod 4, and if so, use the faster algorithm. - if len(p.abs) > 0 && p.abs[0]%4 == 3 { + switch { + case p.abs[0]%4 == 3: + // Check whether p is 3 mod 4, and if so, use the faster algorithm. return z.modSqrt3Mod4Prime(x, p) + case p.abs[0]%8 == 5: + // Check whether p is 5 mod 8, use Atkin's algorithm. + return z.modSqrt5Mod8Prime(x, p) + default: + // Otherwise, use Tonelli-Shanks. + return z.modSqrtTonelliShanks(x, p) } - // Otherwise, use Tonelli-Shanks. - return z.modSqrtTonelliShanks(x, p) } // Lsh sets z = x << n and returns z. diff --git a/src/math/big/int_test.go b/src/math/big/int_test.go index 1ef4d150b8aad..111e2de573c28 100644 --- a/src/math/big/int_test.go +++ b/src/math/big/int_test.go @@ -1360,7 +1360,7 @@ func BenchmarkModSqrt225_Tonelli(b *testing.B) { } } -func BenchmarkModSqrt224_3Mod4(b *testing.B) { +func BenchmarkModSqrt225_3Mod4(b *testing.B) { p := tri(225) x := new(Int).SetUint64(2) for i := 0; i < b.N; i++ { @@ -1369,27 +1369,25 @@ func BenchmarkModSqrt224_3Mod4(b *testing.B) { } } -func BenchmarkModSqrt5430_Tonelli(b *testing.B) { - if isRaceBuilder { - b.Skip("skipping on race builder") - } - p := tri(5430) - x := new(Int).SetUint64(2) +func BenchmarkModSqrt231_Tonelli(b *testing.B) { + p := tri(231) + p.Sub(p, intOne) + p.Sub(p, intOne) // tri(231) - 2 is a prime == 5 mod 8 + x := new(Int).SetUint64(7) for i := 0; i < b.N; i++ { - x.SetUint64(2) + x.SetUint64(7) x.modSqrtTonelliShanks(x, p) } } -func BenchmarkModSqrt5430_3Mod4(b *testing.B) { - if isRaceBuilder { - b.Skip("skipping on race builder") - } - p := tri(5430) - x := new(Int).SetUint64(2) +func BenchmarkModSqrt231_5Mod8(b *testing.B) { + p := tri(231) + p.Sub(p, intOne) + p.Sub(p, intOne) // tri(231) - 2 is a prime == 5 mod 8 + x := new(Int).SetUint64(7) for i := 0; i < b.N; i++ { - x.SetUint64(2) - x.modSqrt3Mod4Prime(x, p) + x.SetUint64(7) + x.modSqrt5Mod8Prime(x, p) } } From fb0d6e4bd18da45fdb2b88640d368e919d3b6c7c Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 30 May 2018 16:55:26 +0200 Subject: [PATCH 036/187] cmd/dist: remove external linking check for macOS 10.6 This was missed in CL 115236. Updates #23122 Change-Id: I5a64bd02d356c21c0e5d02dafafb3721f8dd8e06 Reviewed-on: https://go-review.googlesource.com/115276 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/cmd/dist/test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 5bd5b424afbd6..a1c470cc97cc5 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -889,7 +889,7 @@ func (t *tester) extLink() bool { pair := gohostos + "-" + goarch switch pair { case "android-arm", - "darwin-arm", "darwin-arm64", + "darwin-386", "darwin-amd64", "darwin-arm", "darwin-arm64", "dragonfly-amd64", "freebsd-386", "freebsd-amd64", "freebsd-arm", "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-s390x", @@ -897,15 +897,6 @@ func (t *tester) extLink() bool { "openbsd-386", "openbsd-amd64", "windows-386", "windows-amd64": return true - case "darwin-386", "darwin-amd64": - // linkmode=external fails on OS X 10.6 and earlier == Darwin - // 10.8 and earlier. - unameR, err := exec.Command("uname", "-r").Output() - if err != nil { - log.Fatalf("uname -r: %v", err) - } - major, _ := strconv.Atoi(string(unameR[:bytes.IndexByte(unameR, '.')])) - return major > 10 } return false } From d5bc3b96c6fb758561e6274c8f69232623157ca4 Mon Sep 17 00:00:00 2001 From: Hana Kim Date: Wed, 30 May 2018 10:54:20 -0400 Subject: [PATCH 037/187] cmd/vendor/.../pprof: sync at rev 1ddc9e2 This includes changes in pprof to support - the new -diff_base flag - fix for a bug in handling of legacy Go heap profiles Update #25096 Change-Id: I826ac9244f31cc2c4415388c44a0cbe77303e460 Reviewed-on: https://go-review.googlesource.com/115295 Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- .../google/pprof/internal/driver/cli.go | 49 +++- .../google/pprof/internal/driver/driver.go | 27 +- .../pprof/internal/driver/driver_test.go | 9 +- .../google/pprof/internal/driver/fetch.go | 3 + .../pprof/internal/driver/fetch_test.go | 240 +++++++++++++--- .../pprof/internal/driver/interactive_test.go | 8 +- .../google/pprof/internal/report/report.go | 25 +- .../pprof/internal/report/report_test.go | 118 ++++++++ .../google/pprof/profile/legacy_profile.go | 1 + .../pprof/profile/legacy_profile_test.go | 2 + .../google/pprof/profile/profile.go | 30 ++ .../google/pprof/profile/profile_test.go | 267 +++++++++++++++++- src/cmd/vendor/vendor.json | 72 ++--- 13 files changed, 750 insertions(+), 101 deletions(-) diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go index c3c22e7c96b43..a5153e151132b 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/cli.go @@ -15,6 +15,7 @@ package driver import ( + "errors" "fmt" "os" "strings" @@ -28,6 +29,7 @@ type source struct { ExecName string BuildID string Base []string + DiffBase bool Normalize bool Seconds int @@ -43,7 +45,8 @@ type source struct { func parseFlags(o *plugin.Options) (*source, []string, error) { flag := o.Flagset // Comparisons. - flagBase := flag.StringList("base", "", "Source for base profile for comparison") + flagBase := flag.StringList("base", "", "Source for base profile for profile subtraction") + flagDiffBase := flag.StringList("diff_base", "", "Source for diff base profile for comparison") // Source options. flagSymbolize := flag.String("symbolize", "", "Options for profile symbolization") flagBuildID := flag.String("buildid", "", "Override build id for first mapping") @@ -85,7 +88,7 @@ func parseFlags(o *plugin.Options) (*source, []string, error) { usageMsgVars) }) if len(args) == 0 { - return nil, nil, fmt.Errorf("no profile source specified") + return nil, nil, errors.New("no profile source specified") } var execName string @@ -112,7 +115,7 @@ func parseFlags(o *plugin.Options) (*source, []string, error) { return nil, nil, err } if cmd != nil && *flagHTTP != "" { - return nil, nil, fmt.Errorf("-http is not compatible with an output format on the command line") + return nil, nil, errors.New("-http is not compatible with an output format on the command line") } si := pprofVariables["sample_index"].value @@ -140,15 +143,13 @@ func parseFlags(o *plugin.Options) (*source, []string, error) { Comment: *flagAddComment, } - for _, s := range *flagBase { - if *s != "" { - source.Base = append(source.Base, *s) - } + if err := source.addBaseProfiles(*flagBase, *flagDiffBase); err != nil { + return nil, nil, err } normalize := pprofVariables["normalize"].boolValue() if normalize && len(source.Base) == 0 { - return nil, nil, fmt.Errorf("Must have base profile to normalize by") + return nil, nil, errors.New("must have base profile to normalize by") } source.Normalize = normalize @@ -158,6 +159,34 @@ func parseFlags(o *plugin.Options) (*source, []string, error) { return source, cmd, nil } +// addBaseProfiles adds the list of base profiles or diff base profiles to +// the source. This function will return an error if both base and diff base +// profiles are specified. +func (source *source) addBaseProfiles(flagBase, flagDiffBase []*string) error { + base, diffBase := dropEmpty(flagBase), dropEmpty(flagDiffBase) + if len(base) > 0 && len(diffBase) > 0 { + return errors.New("-base and -diff_base flags cannot both be specified") + } + + source.Base = base + if len(diffBase) > 0 { + source.Base, source.DiffBase = diffBase, true + } + return nil +} + +// dropEmpty list takes a slice of string pointers, and outputs a slice of +// non-empty strings associated with the flag. +func dropEmpty(list []*string) []string { + var l []string + for _, s := range list { + if *s != "" { + l = append(l, *s) + } + } + return l +} + // installFlags creates command line flags for pprof variables. func installFlags(flag plugin.FlagSet) flagsInstalled { f := flagsInstalled{ @@ -240,7 +269,7 @@ func outputFormat(bcmd map[string]*bool, acmd map[string]*string) (cmd []string, for n, b := range bcmd { if *b { if cmd != nil { - return nil, fmt.Errorf("must set at most one output format") + return nil, errors.New("must set at most one output format") } cmd = []string{n} } @@ -248,7 +277,7 @@ func outputFormat(bcmd map[string]*bool, acmd map[string]*string) (cmd []string, for n, s := range acmd { if *s != "" { if cmd != nil { - return nil, fmt.Errorf("must set at most one output format") + return nil, errors.New("must set at most one output format") } cmd = []string{n, *s} } diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go index acc0b4ad8ac99..2dabc3017b57e 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver.go @@ -65,7 +65,13 @@ func generateRawReport(p *profile.Profile, cmd []string, vars variables, o *plug // Identify units of numeric tags in profile. numLabelUnits := identifyNumLabelUnits(p, o.UI) - vars = applyCommandOverrides(cmd, vars) + // Get report output format + c := pprofCommands[cmd[0]] + if c == nil { + panic("unexpected nil command") + } + + vars = applyCommandOverrides(cmd[0], c.format, vars) // Delay focus after configuring report to get percentages on all samples. relative := vars["relative_percentages"].boolValue() @@ -78,10 +84,6 @@ func generateRawReport(p *profile.Profile, cmd []string, vars variables, o *plug if err != nil { return nil, nil, err } - c := pprofCommands[cmd[0]] - if c == nil { - panic("unexpected nil command") - } ropt.OutputFormat = c.format if len(cmd) == 2 { s, err := regexp.Compile(cmd[1]) @@ -149,13 +151,10 @@ func generateReport(p *profile.Profile, cmd []string, vars variables, o *plugin. return out.Close() } -func applyCommandOverrides(cmd []string, v variables) variables { +func applyCommandOverrides(cmd string, outputFormat int, v variables) variables { trim, tagfilter, filter := v["trim"].boolValue(), true, true - switch cmd[0] { - case "proto", "raw": - trim, tagfilter, filter = false, false, false - v.set("addresses", "t") + switch cmd { case "callgrind", "kcachegrind": trim = false v.set("addresses", "t") @@ -163,7 +162,7 @@ func applyCommandOverrides(cmd []string, v variables) variables { trim = false v.set("addressnoinlines", "t") case "peek": - trim, filter = false, false + trim, tagfilter, filter = false, false, false case "list": v.set("nodecount", "0") v.set("lines", "t") @@ -176,6 +175,12 @@ func applyCommandOverrides(cmd []string, v variables) variables { v.set("nodecount", "80") } } + + if outputFormat == report.Proto || outputFormat == report.Raw { + trim, tagfilter, filter = false, false, false + v.set("addresses", "t") + } + if !trim { v.set("nodecount", "0") v.set("nodefraction", "0") diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go index 309e9950b667a..ff6afe9cff7e5 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/driver_test.go @@ -288,7 +288,7 @@ type testFlags struct { floats map[string]float64 strings map[string]string args []string - stringLists map[string][]*string + stringLists map[string][]string } func (testFlags) ExtraUsage() string { return "" } @@ -355,7 +355,12 @@ func (f testFlags) StringVar(p *string, s, d, c string) { func (f testFlags) StringList(s, d, c string) *[]*string { if t, ok := f.stringLists[s]; ok { - return &t + // convert slice of strings to slice of string pointers before returning. + tp := make([]*string, len(t)) + for i, v := range t { + tp[i] = &v + } + return &tp } return &[]*string{} } diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go index 7c576de61498b..7a7a1a20f2a2e 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go @@ -63,6 +63,9 @@ func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) { } if pbase != nil { + if s.DiffBase { + pbase.SetLabel("pprof::base", []string{"true"}) + } if s.Normalize { err := p.Normalize(pbase) if err != nil { diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go index afb135b7cdda1..e67b2e9f87108 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/fetch_test.go @@ -210,13 +210,20 @@ func TestFetchWithBase(t *testing.T) { baseVars := pprofVariables defer func() { pprofVariables = baseVars }() + type WantSample struct { + values []int64 + labels map[string][]string + } + const path = "testdata/" type testcase struct { - desc string - sources []string - bases []string - normalize bool - expectedSamples [][]int64 + desc string + sources []string + bases []string + diffBases []string + normalize bool + wantSamples []WantSample + wantErrorMsg string } testcases := []testcase{ @@ -224,58 +231,216 @@ func TestFetchWithBase(t *testing.T) { "not normalized base is same as source", []string{path + "cppbench.contention"}, []string{path + "cppbench.contention"}, + nil, + false, + nil, + "", + }, + { + "not normalized base is same as source", + []string{path + "cppbench.contention"}, + []string{path + "cppbench.contention"}, + nil, false, - [][]int64{}, + nil, + "", }, { "not normalized single source, multiple base (all profiles same)", []string{path + "cppbench.contention"}, []string{path + "cppbench.contention", path + "cppbench.contention"}, + nil, false, - [][]int64{{-2700, -608881724}, {-100, -23992}, {-200, -179943}, {-100, -17778444}, {-100, -75976}, {-300, -63568134}}, + []WantSample{ + { + values: []int64{-2700, -608881724}, + labels: map[string][]string{}, + }, + { + values: []int64{-100, -23992}, + labels: map[string][]string{}, + }, + { + values: []int64{-200, -179943}, + labels: map[string][]string{}, + }, + { + values: []int64{-100, -17778444}, + labels: map[string][]string{}, + }, + { + values: []int64{-100, -75976}, + labels: map[string][]string{}, + }, + { + values: []int64{-300, -63568134}, + labels: map[string][]string{}, + }, + }, + "", }, { "not normalized, different base and source", []string{path + "cppbench.contention"}, []string{path + "cppbench.small.contention"}, + nil, false, - [][]int64{{1700, 608878600}, {100, 23992}, {200, 179943}, {100, 17778444}, {100, 75976}, {300, 63568134}}, + []WantSample{ + { + values: []int64{1700, 608878600}, + labels: map[string][]string{}, + }, + { + values: []int64{100, 23992}, + labels: map[string][]string{}, + }, + { + values: []int64{200, 179943}, + labels: map[string][]string{}, + }, + { + values: []int64{100, 17778444}, + labels: map[string][]string{}, + }, + { + values: []int64{100, 75976}, + labels: map[string][]string{}, + }, + { + values: []int64{300, 63568134}, + labels: map[string][]string{}, + }, + }, + "", }, { "normalized base is same as source", []string{path + "cppbench.contention"}, []string{path + "cppbench.contention"}, + nil, true, - [][]int64{}, + nil, + "", }, { "normalized single source, multiple base (all profiles same)", []string{path + "cppbench.contention"}, []string{path + "cppbench.contention", path + "cppbench.contention"}, + nil, true, - [][]int64{}, + nil, + "", }, { "normalized different base and source", []string{path + "cppbench.contention"}, []string{path + "cppbench.small.contention"}, + nil, true, - [][]int64{{-229, -370}, {28, 0}, {57, 0}, {28, 80}, {28, 0}, {85, 287}}, + []WantSample{ + { + values: []int64{-229, -370}, + labels: map[string][]string{}, + }, + { + values: []int64{28, 0}, + labels: map[string][]string{}, + }, + { + values: []int64{57, 0}, + labels: map[string][]string{}, + }, + { + values: []int64{28, 80}, + labels: map[string][]string{}, + }, + { + values: []int64{28, 0}, + labels: map[string][]string{}, + }, + { + values: []int64{85, 287}, + labels: map[string][]string{}, + }, + }, + "", + }, + { + "not normalized diff base is same as source", + []string{path + "cppbench.contention"}, + nil, + []string{path + "cppbench.contention"}, + false, + []WantSample{ + { + values: []int64{2700, 608881724}, + labels: map[string][]string{}, + }, + { + values: []int64{100, 23992}, + labels: map[string][]string{}, + }, + { + values: []int64{200, 179943}, + labels: map[string][]string{}, + }, + { + values: []int64{100, 17778444}, + labels: map[string][]string{}, + }, + { + values: []int64{100, 75976}, + labels: map[string][]string{}, + }, + { + values: []int64{300, 63568134}, + labels: map[string][]string{}, + }, + { + values: []int64{-2700, -608881724}, + labels: map[string][]string{"pprof::base": {"true"}}, + }, + { + values: []int64{-100, -23992}, + labels: map[string][]string{"pprof::base": {"true"}}, + }, + { + values: []int64{-200, -179943}, + labels: map[string][]string{"pprof::base": {"true"}}, + }, + { + values: []int64{-100, -17778444}, + labels: map[string][]string{"pprof::base": {"true"}}, + }, + { + values: []int64{-100, -75976}, + labels: map[string][]string{"pprof::base": {"true"}}, + }, + { + values: []int64{-300, -63568134}, + labels: map[string][]string{"pprof::base": {"true"}}, + }, + }, + "", + }, + { + "diff_base and base both specified", + []string{path + "cppbench.contention"}, + []string{path + "cppbench.contention"}, + []string{path + "cppbench.contention"}, + false, + nil, + "-base and -diff_base flags cannot both be specified", }, } for _, tc := range testcases { t.Run(tc.desc, func(t *testing.T) { pprofVariables = baseVars.makeCopy() - - base := make([]*string, len(tc.bases)) - for i, s := range tc.bases { - base[i] = &s - } - f := testFlags{ - stringLists: map[string][]*string{ - "base": base, + stringLists: map[string][]string{ + "base": tc.bases, + "diff_base": tc.diffBases, }, bools: map[string]bool{ "normalize": tc.normalize, @@ -289,30 +454,37 @@ func TestFetchWithBase(t *testing.T) { }) src, _, err := parseFlags(o) + if tc.wantErrorMsg != "" { + if err == nil { + t.Fatalf("got nil, want error %q", tc.wantErrorMsg) + } + + if gotErrMsg := err.Error(); gotErrMsg != tc.wantErrorMsg { + t.Fatalf("got error %q, want error %q", gotErrMsg, tc.wantErrorMsg) + } + return + } + if err != nil { - t.Fatalf("%s: %v", tc.desc, err) + t.Fatalf("got error %q, want no error", err) } p, err := fetchProfiles(src, o) - pprofVariables = baseVars + if err != nil { - t.Fatal(err) + t.Fatalf("got error %q, want no error", err) } - if want, got := len(tc.expectedSamples), len(p.Sample); want != got { - t.Fatalf("want %d samples got %d", want, got) + if got, want := len(p.Sample), len(tc.wantSamples); got != want { + t.Fatalf("got %d samples want %d", got, want) } - if len(p.Sample) > 0 { - for i, sample := range p.Sample { - if want, got := len(tc.expectedSamples[i]), len(sample.Value); want != got { - t.Errorf("want %d values for sample %d, got %d", want, i, got) - } - for j, value := range sample.Value { - if want, got := tc.expectedSamples[i][j], value; want != got { - t.Errorf("want value of %d for value %d of sample %d, got %d", want, j, i, got) - } - } + for i, sample := range p.Sample { + if !reflect.DeepEqual(tc.wantSamples[i].values, sample.Value) { + t.Errorf("for sample %d got values %v, want %v", i, sample.Value, tc.wantSamples[i]) + } + if !reflect.DeepEqual(tc.wantSamples[i].labels, sample.Label) { + t.Errorf("for sample %d got labels %v, want %v", i, sample.Label, tc.wantSamples[i].labels) } } }) diff --git a/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go b/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go index db26862c7d1cc..8d775e16bdbc0 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/driver/interactive_test.go @@ -294,7 +294,13 @@ func TestInteractiveCommands(t *testing.T) { t.Errorf("failed on %q: %v", tc.input, err) continue } - vars = applyCommandOverrides(cmd, vars) + + // Get report output format + c := pprofCommands[cmd[0]] + if c == nil { + t.Errorf("unexpected nil command") + } + vars = applyCommandOverrides(cmd[0], c.format, vars) for n, want := range tc.want { if got := vars[n].stringValue(); got != want { diff --git a/src/cmd/vendor/github.com/google/pprof/internal/report/report.go b/src/cmd/vendor/github.com/google/pprof/internal/report/report.go index 15cadfb548069..76db9cbf991e5 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/report/report.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/report/report.go @@ -264,6 +264,10 @@ func (rpt *Report) newGraph(nodes graph.NodeSet) *graph.Graph { s.NumUnit = numUnits } + // Remove label marking samples from the base profiles, so it does not appear + // as a nodelet in the graph view. + prof.RemoveLabel("pprof::base") + formatTag := func(v int64, key string) string { return measurement.ScaledLabel(v, key, o.OutputUnit) } @@ -1212,10 +1216,11 @@ func NewDefault(prof *profile.Profile, options Options) *Report { return New(prof, o) } -// computeTotal computes the sum of all sample values. This will be -// used to compute percentages. +// computeTotal computes the sum of the absolute value of all sample values. +// If any samples have the label "pprof::base" with value "true", then the total +// will only include samples with that label. func computeTotal(prof *profile.Profile, value, meanDiv func(v []int64) int64) int64 { - var div, ret int64 + var div, total, diffDiv, diffTotal int64 for _, sample := range prof.Sample { var d, v int64 v = value(sample.Value) @@ -1225,13 +1230,21 @@ func computeTotal(prof *profile.Profile, value, meanDiv func(v []int64) int64) i if v < 0 { v = -v } - ret += v + total += v div += d + if sample.HasLabel("pprof::base", "true") { + diffTotal += v + diffDiv += d + } + } + if diffTotal > 0 { + total = diffTotal + div = diffDiv } if div != 0 { - return ret / div + return total / div } - return ret + return total } // Report contains the data and associated routines to extract a diff --git a/src/cmd/vendor/github.com/google/pprof/internal/report/report_test.go b/src/cmd/vendor/github.com/google/pprof/internal/report/report_test.go index 49c6e4934f5c2..9eb435bbb816d 100644 --- a/src/cmd/vendor/github.com/google/pprof/internal/report/report_test.go +++ b/src/cmd/vendor/github.com/google/pprof/internal/report/report_test.go @@ -287,3 +287,121 @@ func TestLegendActiveFilters(t *testing.T) { } } } + +func TestComputeTotal(t *testing.T) { + p1 := testProfile.Copy() + p1.Sample = []*profile.Sample{ + { + Location: []*profile.Location{testL[0]}, + Value: []int64{1, 1}, + }, + { + Location: []*profile.Location{testL[2], testL[1], testL[0]}, + Value: []int64{1, 10}, + }, + { + Location: []*profile.Location{testL[4], testL[2], testL[0]}, + Value: []int64{1, 100}, + }, + } + + p2 := testProfile.Copy() + p2.Sample = []*profile.Sample{ + { + Location: []*profile.Location{testL[0]}, + Value: []int64{1, 1}, + }, + { + Location: []*profile.Location{testL[2], testL[1], testL[0]}, + Value: []int64{1, -10}, + }, + { + Location: []*profile.Location{testL[4], testL[2], testL[0]}, + Value: []int64{1, 100}, + }, + } + + p3 := testProfile.Copy() + p3.Sample = []*profile.Sample{ + { + Location: []*profile.Location{testL[0]}, + Value: []int64{10000, 1}, + }, + { + Location: []*profile.Location{testL[2], testL[1], testL[0]}, + Value: []int64{-10, 3}, + Label: map[string][]string{"pprof::base": {"true"}}, + }, + { + Location: []*profile.Location{testL[2], testL[1], testL[0]}, + Value: []int64{1000, -10}, + }, + { + Location: []*profile.Location{testL[2], testL[1], testL[0]}, + Value: []int64{-9000, 3}, + Label: map[string][]string{"pprof::base": {"true"}}, + }, + { + Location: []*profile.Location{testL[2], testL[1], testL[0]}, + Value: []int64{-1, 3}, + Label: map[string][]string{"pprof::base": {"true"}}, + }, + { + Location: []*profile.Location{testL[4], testL[2], testL[0]}, + Value: []int64{100, 100}, + }, + { + Location: []*profile.Location{testL[2], testL[1], testL[0]}, + Value: []int64{100, 3}, + Label: map[string][]string{"pprof::base": {"true"}}, + }, + } + + testcases := []struct { + desc string + prof *profile.Profile + value, meanDiv func(v []int64) int64 + wantTotal int64 + }{ + { + desc: "no diff base, all positive values, index 1", + prof: p1, + value: func(v []int64) int64 { + return v[0] + }, + wantTotal: 3, + }, + { + desc: "no diff base, all positive values, index 2", + prof: p1, + value: func(v []int64) int64 { + return v[1] + }, + wantTotal: 111, + }, + { + desc: "no diff base, some negative values", + prof: p2, + value: func(v []int64) int64 { + return v[1] + }, + wantTotal: 111, + }, + { + desc: "diff base, some negative values", + prof: p3, + value: func(v []int64) int64 { + return v[0] + }, + wantTotal: 9111, + }, + } + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + if gotTotal := computeTotal(tc.prof, tc.value, tc.meanDiv); gotTotal != tc.wantTotal { + t.Errorf("got total %d, want %v", gotTotal, tc.wantTotal) + } + }) + } +} diff --git a/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile.go b/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile.go index 096890d9b4878..0c8f3bb5b71f4 100644 --- a/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile.go +++ b/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile.go @@ -1103,6 +1103,7 @@ var heapzSampleTypes = [][]string{ {"objects", "space"}, {"inuse_objects", "inuse_space"}, {"alloc_objects", "alloc_space"}, + {"alloc_objects", "alloc_space", "inuse_objects", "inuse_space"}, // Go pprof legacy profiles } var contentionzSampleTypes = [][]string{ {"contentions", "delay"}, diff --git a/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile_test.go b/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile_test.go index 5f63453e09d0f..6ba0e338c9bd4 100644 --- a/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile_test.go +++ b/src/cmd/vendor/github.com/google/pprof/profile/legacy_profile_test.go @@ -39,10 +39,12 @@ func TestLegacyProfileType(t *testing.T) { {[]string{"objects", "space"}, heap, true, "heapzSampleTypes"}, {[]string{"inuse_objects", "inuse_space"}, heap, true, "heapzSampleTypes"}, {[]string{"alloc_objects", "alloc_space"}, heap, true, "heapzSampleTypes"}, + {[]string{"alloc_objects", "alloc_space", "inuse_objects", "inuse_space"}, heap, true, "heapzSampleTypes"}, {[]string{"contentions", "delay"}, cont, true, "contentionzSampleTypes"}, // False cases {[]string{"objects"}, heap, false, "heapzSampleTypes"}, {[]string{"objects", "unknown"}, heap, false, "heapzSampleTypes"}, + {[]string{"inuse_objects", "inuse_space", "alloc_objects", "alloc_space"}, heap, false, "heapzSampleTypes"}, {[]string{"contentions", "delay"}, heap, false, "heapzSampleTypes"}, {[]string{"samples", "cpu"}, heap, false, "heapzSampleTypes"}, {[]string{"samples", "cpu"}, cont, false, "contentionzSampleTypes"}, diff --git a/src/cmd/vendor/github.com/google/pprof/profile/profile.go b/src/cmd/vendor/github.com/google/pprof/profile/profile.go index 350538bf432b8..452194b12788a 100644 --- a/src/cmd/vendor/github.com/google/pprof/profile/profile.go +++ b/src/cmd/vendor/github.com/google/pprof/profile/profile.go @@ -674,6 +674,36 @@ func numLabelsToString(numLabels map[string][]int64, numUnits map[string][]strin return strings.Join(ls, " ") } +// SetLabel sets the specified key to the specified value for all samples in the +// profile. +func (p *Profile) SetLabel(key string, value []string) { + for _, sample := range p.Sample { + if sample.Label == nil { + sample.Label = map[string][]string{key: value} + } else { + sample.Label[key] = value + } + } +} + +// RemoveLabel removes all labels associated with the specified key for all +// samples in the profile. +func (p *Profile) RemoveLabel(key string) { + for _, sample := range p.Sample { + delete(sample.Label, key) + } +} + +// HasLabel returns true if a sample has a label with indicated key and value. +func (s *Sample) HasLabel(key, value string) bool { + for _, v := range s.Label[key] { + if v == value { + return true + } + } + return false +} + // Scale multiplies all sample values in a profile by a constant. func (p *Profile) Scale(ratio float64) { if ratio == 1 { diff --git a/src/cmd/vendor/github.com/google/pprof/profile/profile_test.go b/src/cmd/vendor/github.com/google/pprof/profile/profile_test.go index 8ed67b1dd6c0e..5b299b1d55263 100644 --- a/src/cmd/vendor/github.com/google/pprof/profile/profile_test.go +++ b/src/cmd/vendor/github.com/google/pprof/profile/profile_test.go @@ -741,7 +741,7 @@ func TestNumLabelMerge(t *testing.T) { wantNumUnits []map[string][]string }{ { - name: "different tag units not merged", + name: "different label units not merged", profs: []*Profile{testProfile4.Copy(), testProfile5.Copy()}, wantNumLabels: []map[string][]int64{ { @@ -912,6 +912,271 @@ func locationHash(s *Sample) string { return tb } +func TestHasLabel(t *testing.T) { + var testcases = []struct { + desc string + labels map[string][]string + key string + value string + wantHasLabel bool + }{ + { + desc: "empty label does not have label", + labels: map[string][]string{}, + key: "key", + value: "value", + wantHasLabel: false, + }, + { + desc: "label with one key and value has label", + labels: map[string][]string{"key": {"value"}}, + key: "key", + value: "value", + wantHasLabel: true, + }, + { + desc: "label with one key and value does not have label", + labels: map[string][]string{"key": {"value"}}, + key: "key1", + value: "value1", + wantHasLabel: false, + }, + { + desc: "label with many keys and values has label", + labels: map[string][]string{ + "key1": {"value2", "value1"}, + "key2": {"value1", "value2", "value2"}, + "key3": {"value1", "value2", "value2"}, + }, + key: "key1", + value: "value1", + wantHasLabel: true, + }, + { + desc: "label with many keys and values does not have label", + labels: map[string][]string{ + "key1": {"value2", "value1"}, + "key2": {"value1", "value2", "value2"}, + "key3": {"value1", "value2", "value2"}, + }, + key: "key5", + value: "value5", + wantHasLabel: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + sample := &Sample{ + Label: tc.labels, + } + if gotHasLabel := sample.HasLabel(tc.key, tc.value); gotHasLabel != tc.wantHasLabel { + t.Errorf("sample.HasLabel(%q, %q) got %v, want %v", tc.key, tc.value, gotHasLabel, tc.wantHasLabel) + } + }) + } +} + +func TestRemove(t *testing.T) { + var testcases = []struct { + desc string + samples []*Sample + removeKey string + wantLabels []map[string][]string + }{ + { + desc: "some samples have label already", + samples: []*Sample{ + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + }, + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key1": {"value1", "value2", "value3"}, + "key2": {"value1"}, + }, + }, + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key1": {"value2"}, + }, + }, + }, + removeKey: "key1", + wantLabels: []map[string][]string{ + {}, + {"key2": {"value1"}}, + {}, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + profile := testProfile1.Copy() + profile.Sample = tc.samples + profile.RemoveLabel(tc.removeKey) + if got, want := len(profile.Sample), len(tc.wantLabels); got != want { + t.Fatalf("got %v samples, want %v samples", got, want) + } + for i, sample := range profile.Sample { + wantLabels := tc.wantLabels[i] + if got, want := len(sample.Label), len(wantLabels); got != want { + t.Errorf("got %v label keys for sample %v, want %v", got, i, want) + continue + } + for wantKey, wantValues := range wantLabels { + if gotValues, ok := sample.Label[wantKey]; ok { + if !reflect.DeepEqual(gotValues, wantValues) { + t.Errorf("for key %s, got values %v, want values %v", wantKey, gotValues, wantValues) + } + } else { + t.Errorf("for key %s got no values, want %v", wantKey, wantValues) + } + } + } + }) + } +} + +func TestSetLabel(t *testing.T) { + var testcases = []struct { + desc string + samples []*Sample + setKey string + setVal []string + wantLabels []map[string][]string + }{ + { + desc: "some samples have label already", + samples: []*Sample{ + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + }, + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key1": {"value1", "value2", "value3"}, + "key2": {"value1"}, + }, + }, + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key1": {"value2"}, + }, + }, + }, + setKey: "key1", + setVal: []string{"value1"}, + wantLabels: []map[string][]string{ + {"key1": {"value1"}}, + {"key1": {"value1"}, "key2": {"value1"}}, + {"key1": {"value1"}}, + }, + }, + { + desc: "no samples have labels", + samples: []*Sample{ + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + }, + }, + setKey: "key1", + setVal: []string{"value1"}, + wantLabels: []map[string][]string{ + {"key1": {"value1"}}, + }, + }, + { + desc: "all samples have some labels, but not key being added", + samples: []*Sample{ + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key2": {"value2"}, + }, + }, + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key3": {"value3"}, + }, + }, + }, + setKey: "key1", + setVal: []string{"value1"}, + wantLabels: []map[string][]string{ + {"key1": {"value1"}, "key2": {"value2"}}, + {"key1": {"value1"}, "key3": {"value3"}}, + }, + }, + { + desc: "all samples have key being added", + samples: []*Sample{ + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key1": {"value1"}, + }, + }, + { + Location: []*Location{cpuL[0]}, + Value: []int64{1000}, + Label: map[string][]string{ + "key1": {"value1"}, + }, + }, + }, + setKey: "key1", + setVal: []string{"value1"}, + wantLabels: []map[string][]string{ + {"key1": {"value1"}}, + {"key1": {"value1"}}, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + profile := testProfile1.Copy() + profile.Sample = tc.samples + profile.SetLabel(tc.setKey, tc.setVal) + if got, want := len(profile.Sample), len(tc.wantLabels); got != want { + t.Fatalf("got %v samples, want %v samples", got, want) + } + for i, sample := range profile.Sample { + wantLabels := tc.wantLabels[i] + if got, want := len(sample.Label), len(wantLabels); got != want { + t.Errorf("got %v label keys for sample %v, want %v", got, i, want) + continue + } + for wantKey, wantValues := range wantLabels { + if gotValues, ok := sample.Label[wantKey]; ok { + if !reflect.DeepEqual(gotValues, wantValues) { + t.Errorf("for key %s, got values %v, want values %v", wantKey, gotValues, wantValues) + } + } else { + t.Errorf("for key %s got no values, want %v", wantKey, wantValues) + } + } + } + }) + } +} + func TestNumLabelUnits(t *testing.T) { var tagFilterTests = []struct { desc string diff --git a/src/cmd/vendor/vendor.json b/src/cmd/vendor/vendor.json index 89506e7fe1724..26b32692b28b1 100644 --- a/src/cmd/vendor/vendor.json +++ b/src/cmd/vendor/vendor.json @@ -5,98 +5,98 @@ { "checksumSHA1": "G9UsR+iruMWxwUefhy+ID+VIFNs=", "path": "github.com/google/pprof/driver", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "LzGfApA19baVJIbQEqziWpRS3zE=", "path": "github.com/google/pprof/internal/binutils", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { - "checksumSHA1": "f7aprpcWR7iZX1PJgKBZrpt++XY=", + "checksumSHA1": "uoKLYk9VTOx2kYV3hU3vOGm4BX8=", "path": "github.com/google/pprof/internal/driver", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "IhuyU2pFSHhQxzadDBw1nHbcsrY=", "path": "github.com/google/pprof/internal/elfexec", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "8vah+aXLGpbtn55JR8MkCAEOMrk=", "path": "github.com/google/pprof/internal/graph", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "QPWfnT5pEU2jOOb8l8hpiFzQJ7Q=", "path": "github.com/google/pprof/internal/measurement", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "PWZdFtGfGz/zbQTfvel9737NZdY=", "path": "github.com/google/pprof/internal/plugin", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "LmDglu/S6vFmgqkxubKDZemFHaY=", "path": "github.com/google/pprof/internal/proftest", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { - "checksumSHA1": "gdyWnzbjgwmqJ2EN/WAp+QPu7d0=", + "checksumSHA1": "qgsLCrPLve6es8A3bA3qv2LPoYk=", "path": "github.com/google/pprof/internal/report", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "rWdirHgJi1+TdRwv5v3zjgFKcJA=", "path": "github.com/google/pprof/internal/symbolizer", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "5lS2AF207MVYyjF+82qHkWK2V64=", "path": "github.com/google/pprof/internal/symbolz", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { - "checksumSHA1": "RvwtpZ+NVtPRCo4EiFvLFFHpoBo=", + "checksumSHA1": "JMf63Fn5hz7JFgz6A2aT9DP/bL0=", "path": "github.com/google/pprof/profile", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "xmqfYca88U2c/I4642r3ps9uIRg=", "path": "github.com/google/pprof/third_party/d3", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "LzWzD56Trzpq+0hLR00Yw5Gpepw=", "path": "github.com/google/pprof/third_party/d3flamegraph", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { "checksumSHA1": "738v1E0v0qRW6oAKdCpBEtyVNnY=", "path": "github.com/google/pprof/third_party/svgpan", - "revision": "4d67f66d7c9469639518a80f378434bb7e9156b7", - "revisionTime": "2018-05-09T15:07:09Z" + "revision": "1ddc9e21322e23449cb6709652bf3583969ca167", + "revisionTime": "2018-05-30T14:24:47Z" }, { - "checksumSHA1": "UDJQBwUTuQYEHHJ/D7nPBv1qNqI=", + "checksumSHA1": "J5yI4NzHbondzccJmummyJR/kQQ=", "path": "github.com/ianlancetaylor/demangle", - "revision": "4883227f66371e02c4948937d3e2be1664d9be38", - "revisionTime": "2016-09-27T19:13:59Z" + "revision": "fc6590592b44fedfff586c5d94647c090fbd6bac", + "revisionTime": "2018-05-24T22:59:00Z" }, { "path": "golang.org/x/arch/arm/armasm", From 31e1c30f55165785dd12e7c67babedeb950a721d Mon Sep 17 00:00:00 2001 From: David Chase Date: Fri, 25 May 2018 16:08:13 -0400 Subject: [PATCH 038/187] cmd/compile: do not allow regalloc to LoadReg G register On architectures where G is stored in a register, it is possible for a variable to allocated to it, and subsequently that variable may be spilled and reloaded, for example because of an intervening call. If such an allocation reaches a join point and it is the primary predecessor, it becomes the target of a reload, which is only usually right. Fix: guard all the LoadReg ops, and spill value in the G register (if any) before merges (in the same way that 387 FP registers are freed between blocks). Includes test. Fixes #25504. Change-Id: I0482a53e20970c7315bf09c0e407ae5bba2fe05d Reviewed-on: https://go-review.googlesource.com/114695 Run-TryBot: David Chase TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/export_test.go | 1 + src/cmd/compile/internal/ssa/regalloc.go | 17 +++++++ src/cmd/compile/internal/ssa/regalloc_test.go | 49 +++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/src/cmd/compile/internal/ssa/export_test.go b/src/cmd/compile/internal/ssa/export_test.go index be9f19b51cbc1..5832050a8a608 100644 --- a/src/cmd/compile/internal/ssa/export_test.go +++ b/src/cmd/compile/internal/ssa/export_test.go @@ -28,6 +28,7 @@ var testCtxts = map[string]*obj.Link{ func testConfig(tb testing.TB) *Conf { return testConfigArch(tb, "amd64") } func testConfigS390X(tb testing.TB) *Conf { return testConfigArch(tb, "s390x") } +func testConfigARM64(tb testing.TB) *Conf { return testConfigArch(tb, "arm64") } func testConfigArch(tb testing.TB, arch string) *Conf { ctxt, ok := testCtxts[arch] diff --git a/src/cmd/compile/internal/ssa/regalloc.go b/src/cmd/compile/internal/ssa/regalloc.go index 080ad0fda11ca..bbf1932981458 100644 --- a/src/cmd/compile/internal/ssa/regalloc.go +++ b/src/cmd/compile/internal/ssa/regalloc.go @@ -532,6 +532,9 @@ func (s *regAllocState) allocValToReg(v *Value, mask regMask, nospill bool, pos } s.assignReg(r, v, c) + if c.Op == OpLoadReg && s.isGReg(r) { + s.f.Fatalf("allocValToReg.OpLoadReg targeting g: " + c.LongString()) + } if nospill { s.nospill |= regMask(1) << r } @@ -809,6 +812,10 @@ func (s *regAllocState) regspec(op Op) regInfo { return opcodeTable[op].reg } +func (s *regAllocState) isGReg(r register) bool { + return s.f.Config.hasGReg && s.GReg == r +} + func (s *regAllocState) regalloc(f *Func) { regValLiveSet := f.newSparseSet(f.NumValues()) // set of values that may be live in register defer f.retSparseSet(regValLiveSet) @@ -951,6 +958,7 @@ func (s *regAllocState) regalloc(f *Func) { // Majority vote? Deepest nesting level? phiRegs = phiRegs[:0] var phiUsed regMask + for _, v := range phis { if !s.values[v.ID].needReg { phiRegs = append(phiRegs, noRegister) @@ -1516,6 +1524,9 @@ func (s *regAllocState) regalloc(f *Func) { // predecessor of it, find live values that we use soon after // the merge point and promote them to registers now. if len(b.Succs) == 1 { + if s.f.Config.hasGReg && s.regs[s.GReg].v != nil { + s.freeReg(s.GReg) // Spill value in G register before any merge. + } // For this to be worthwhile, the loop must have no calls in it. top := b.Succs[0].b loop := s.loopnest.b2l[top.ID] @@ -1996,6 +2007,9 @@ func (e *edgeState) process() { c = e.p.NewValue1(pos, OpLoadReg, c.Type, c) } e.set(r, vid, c, false, pos) + if c.Op == OpLoadReg && e.s.isGReg(register(r.(*Register).num)) { + e.s.f.Fatalf("process.OpLoadReg targeting g: " + c.LongString()) + } } } @@ -2110,6 +2124,9 @@ func (e *edgeState) processDest(loc Location, vid ID, splice **Value, pos src.XP } } e.set(loc, vid, x, true, pos) + if x.Op == OpLoadReg && e.s.isGReg(register(loc.(*Register).num)) { + e.s.f.Fatalf("processDest.OpLoadReg targeting g: " + x.LongString()) + } if splice != nil { (*splice).Uses-- *splice = x diff --git a/src/cmd/compile/internal/ssa/regalloc_test.go b/src/cmd/compile/internal/ssa/regalloc_test.go index 02751a9349aae..bb8be5e7ac6f6 100644 --- a/src/cmd/compile/internal/ssa/regalloc_test.go +++ b/src/cmd/compile/internal/ssa/regalloc_test.go @@ -36,6 +36,55 @@ func TestLiveControlOps(t *testing.T) { checkFunc(f.f) } +// Test to make sure G register is never reloaded from spill (spill of G is okay) +// See #25504 +func TestNoGetgLoadReg(t *testing.T) { + /* + Original: + func fff3(i int) *g { + gee := getg() + if i == 0 { + fff() + } + return gee // here + } + */ + c := testConfigARM64(t) + f := c.Fun("b1", + Bloc("b1", + Valu("v1", OpInitMem, types.TypeMem, 0, nil), + Valu("v6", OpArg, c.config.Types.Int64, 0, c.Frontend().Auto(src.NoXPos, c.config.Types.Int64)), + Valu("v8", OpGetG, c.config.Types.Int64.PtrTo(), 0, nil, "v1"), + Valu("v11", OpARM64CMPconst, types.TypeFlags, 0, nil, "v6"), + Eq("v11", "b2", "b4"), + ), + Bloc("b4", + Goto("b3"), + ), + Bloc("b3", + Valu("v14", OpPhi, types.TypeMem, 0, nil, "v1", "v12"), + Valu("sb", OpSB, c.config.Types.Uintptr, 0, nil), + Valu("v16", OpARM64MOVDstore, types.TypeMem, 0, nil, "v8", "sb", "v14"), + Exit("v16"), + ), + Bloc("b2", + Valu("v12", OpARM64CALLstatic, types.TypeMem, 0, nil, "v1"), + Goto("b3"), + ), + ) + regalloc(f.f) + checkFunc(f.f) + // Double-check that we never restore to the G register. Regalloc should catch it, but check again anyway. + r := f.f.RegAlloc + for _, b := range f.blocks { + for _, v := range b.Values { + if v.Op == OpLoadReg && r[v.ID].String() == "g" { + t.Errorf("Saw OpLoadReg targeting g register: %s", v.LongString()) + } + } + } +} + // Test to make sure we don't push spills into loops. // See issue #19595. func TestSpillWithLoop(t *testing.T) { From b12e341616365cafd6f7eab9aaaa29c4155c1a76 Mon Sep 17 00:00:00 2001 From: Johan Brandhorst Date: Wed, 30 May 2018 17:11:31 +0000 Subject: [PATCH 039/187] net/http: add js/wasm compatible DefaultTransport Adds a new Transport type for the js/wasm target that uses the JavaScript Fetch API for sending HTTP requests. Support for streaming response bodies is used when available, falling back to reading the entire response into memory at once. Updates #25506 Change-Id: Ie9ea433a1a2ed2f65b03c6cc84a16e70c06fcf5c GitHub-Last-Rev: 6df646745b8e0474781f4b1a3084536e573e8e8c GitHub-Pull-Request: golang/go#25550 Reviewed-on: https://go-review.googlesource.com/114515 Reviewed-by: Brad Fitzpatrick --- src/go/build/deps_test.go | 1 + src/net/http/roundtrip.go | 15 ++ src/net/http/roundtrip_js.go | 267 +++++++++++++++++++++++++++++++++++ src/net/http/transport.go | 7 +- 4 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 src/net/http/roundtrip.go create mode 100644 src/net/http/roundtrip_js.go diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index 5137ccfe3f8ae..ce674351deaba 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -411,6 +411,7 @@ var pkgDeps = map[string][]string{ "net/http/httptrace", "net/http/internal", "runtime/debug", + "syscall/js", }, "net/http/internal": {"L4"}, "net/http/httptrace": {"context", "crypto/tls", "internal/nettrace", "net", "reflect", "time"}, diff --git a/src/net/http/roundtrip.go b/src/net/http/roundtrip.go new file mode 100644 index 0000000000000..c8e691cc46038 --- /dev/null +++ b/src/net/http/roundtrip.go @@ -0,0 +1,15 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !js !wasm + +package http + +// RoundTrip implements the RoundTripper interface. +// +// For higher-level HTTP client support (such as handling of cookies +// and redirects), see Get, Post, and the Client type. +func (t *Transport) RoundTrip(req *Request) (*Response, error) { + return t.roundTrip(req) +} diff --git a/src/net/http/roundtrip_js.go b/src/net/http/roundtrip_js.go new file mode 100644 index 0000000000000..e60b7368df727 --- /dev/null +++ b/src/net/http/roundtrip_js.go @@ -0,0 +1,267 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package http + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "strconv" + "syscall/js" +) + +// RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. +func (*Transport) RoundTrip(req *Request) (*Response, error) { + if useFakeNetwork(req) { + return t.roundTrip(req) + } + headers := js.Global.Get("Headers").New() + for key, values := range req.Header { + for _, value := range values { + headers.Call("append", key, value) + } + } + + ac := js.Global.Get("AbortController") + if ac != js.Undefined { + // Some browsers that support WASM don't necessarily support + // the AbortController. See + // https://developer.mozilla.org/en-US/docs/Web/API/AbortController#Browser_compatibility. + ac = ac.New() + } + + opt := js.Global.Get("Object").New() + // See https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch + // for options available. + opt.Set("headers", headers) + opt.Set("method", req.Method) + opt.Set("credentials", "same-origin") + if ac != js.Undefined { + opt.Set("signal", ac.Get("signal")) + } + + if req.Body != nil { + // TODO(johanbrandhorst): Stream request body when possible. + // See https://bugs.chromium.org/p/chromium/issues/detail?id=688906 for Blink issue. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1387483 for Firefox issue. + // See https://github.com/web-platform-tests/wpt/issues/7693 for WHATWG tests issue. + // See https://developer.mozilla.org/en-US/docs/Web/API/Streams_API for more details on the Streams API + // and browser support. + body, err := ioutil.ReadAll(req.Body) + if err != nil { + req.Body.Close() // RoundTrip must always close the body, including on errors. + return nil, err + } + req.Body.Close() + opt.Set("body", body) + } + respPromise := js.Global.Call("fetch", req.URL.String(), opt) + var ( + respCh = make(chan *Response, 1) + errCh = make(chan error, 1) + ) + success := js.NewCallback(func(args []js.Value) { + result := args[0] + header := Header{} + // https://developer.mozilla.org/en-US/docs/Web/API/Headers/entries + headersIt := result.Get("headers").Call("entries") + for { + n := headersIt.Call("next") + if n.Get("done").Bool() { + break + } + pair := n.Get("value") + key, value := pair.Index(0).String(), pair.Index(1).String() + ck := CanonicalHeaderKey(key) + header[ck] = append(header[ck], value) + } + + contentLength := int64(0) + if cl, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64); err == nil { + contentLength = cl + } + + b := result.Get("body") + var body io.ReadCloser + if b != js.Undefined { + body = &streamReader{stream: b.Call("getReader")} + } else { + // Fall back to using ArrayBuffer + // https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer + body = &arrayReader{arrayPromise: result.Call("arrayBuffer")} + } + + select { + case respCh <- &Response{ + Status: result.Get("status").String() + " " + StatusText(result.Get("status").Int()), + StatusCode: result.Get("status").Int(), + Header: header, + ContentLength: contentLength, + Body: body, + Request: req, + }: + case <-req.Context().Done(): + } + }) + defer success.Close() + failure := js.NewCallback(func(args []js.Value) { + err := fmt.Errorf("net/http: fetch() failed: %s", args[0].String()) + select { + case errCh <- err: + case <-req.Context().Done(): + } + }) + defer failure.Close() + respPromise.Call("then", success, failure) + select { + case <-req.Context().Done(): + if ac != js.Undefined { + // Abort the Fetch request + ac.Call("abort") + } + return nil, req.Context().Err() + case resp := <-respCh: + return resp, nil + case err := <-errCh: + return nil, err + } +} + +// useFakeNetwork is used to determine whether the request is made +// by a test and should be made to use the fake in-memory network. +func useFakeNetwork(req *Request) bool { + host, _, err := net.SplitHostPort(req.Host) + if err != nil { + host = req.Host + } + if ip := net.ParseIP(host); ip != nil { + return ip.IsLoopback(ip) + } + return host == "localhost" +} + +// streamReader implements an io.ReadCloser wrapper for ReadableStream. +// See https://fetch.spec.whatwg.org/#readablestream for more information. +type streamReader struct { + pending []byte + stream js.Value + err error // sticky read error +} + +func (r *streamReader) Read(p []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + if len(r.pending) == 0 { + var ( + bCh = make(chan []byte, 1) + errCh = make(chan error, 1) + ) + success := js.NewCallback(func(args []js.Value) { + result := args[0] + if result.Get("done").Bool() { + errCh <- io.EOF + return + } + value := make([]byte, result.Get("value").Get("byteLength").Int()) + js.ValueOf(value).Call("set", result.Get("value")) + bCh <- value + }) + defer success.Close() + failure := js.NewCallback(func(args []js.Value) { + // Assumes it's a TypeError. See + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError + // for more information on this type. See + // https://streams.spec.whatwg.org/#byob-reader-read for the spec on + // the read method. + errCh <- errors.New(args[0].Get("message").String()) + }) + defer failure.Close() + r.stream.Call("read").Call("then", success, failure) + select { + case b := <-bCh: + r.pending = b + case err := <-errCh: + r.err = err + return 0, err + } + } + n = copy(p, r.pending) + r.pending = r.pending[n:] + return n, nil +} + +func (r *streamReader) Close() error { + // This ignores any error returned from cancel method. So far, I did not encounter any concrete + // situation where reporting the error is meaningful. Most users ignore error from resp.Body.Close(). + // If there's a need to report error here, it can be implemented and tested when that need comes up. + r.stream.Call("cancel") + if r.err == nil { + r.err = errClosed + } + return nil +} + +// arrayReader implements an io.ReadCloser wrapper for ArrayBuffer. +// https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer. +type arrayReader struct { + arrayPromise js.Value + pending []byte + read bool + err error // sticky read error +} + +func (r *arrayReader) Read(p []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + if !r.read { + r.read = true + var ( + bCh = make(chan []byte, 1) + errCh = make(chan error, 1) + ) + success := js.NewCallback(func(args []js.Value) { + // Wrap the input ArrayBuffer with a Uint8Array + uint8arrayWrapper := js.Global.Get("Uint8Array").New(args[0]) + value := make([]byte, uint8arrayWrapper.Get("byteLength").Int()) + js.ValueOf(value).Call("set", uint8arrayWrapper) + bCh <- value + }) + defer success.Close() + failure := js.NewCallback(func(args []js.Value) { + // Assumes it's a TypeError. See + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError + // for more information on this type. + // See https://fetch.spec.whatwg.org/#concept-body-consume-body for reasons this might error. + errCh <- errors.New(args[0].Get("message").String()) + }) + defer failure.Close() + r.arrayPromise.Call("then", success, failure) + select { + case b := <-bCh: + r.pending = b + case err := <-errCh: + return 0, err + } + } + if len(r.pending) == 0 { + return 0, io.EOF + } + n = copy(p, r.pending) + r.pending = r.pending[n:] + return n, nil +} + +func (r *arrayReader) Close() error { + if r.err == nil { + r.err = errClosed + } + return nil +} diff --git a/src/net/http/transport.go b/src/net/http/transport.go index 5bf9ff951f52c..731bf176a8723 100644 --- a/src/net/http/transport.go +++ b/src/net/http/transport.go @@ -311,11 +311,8 @@ func (tr *transportRequest) setError(err error) { tr.mu.Unlock() } -// RoundTrip implements the RoundTripper interface. -// -// For higher-level HTTP client support (such as handling of cookies -// and redirects), see Get, Post, and the Client type. -func (t *Transport) RoundTrip(req *Request) (*Response, error) { +// roundTrip implements a RoundTripper over HTTP. +func (t *Transport) roundTrip(req *Request) (*Response, error) { t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) ctx := req.Context() trace := httptrace.ContextClientTrace(ctx) From b9ecac03cb8084bb24c9a75e0f02e164292bb427 Mon Sep 17 00:00:00 2001 From: teague Date: Wed, 30 May 2018 11:35:11 -0400 Subject: [PATCH 040/187] net/http/httputil: reduced log verbosity in reverseproxy_test.go For functions TestClonesRequestHeaders and TestReverseProxy_PanicBodyError, I made changes to update the log verbosity. Fixes #25634 Change-Id: I2a0ef70a8191cfb1a0005949345be722fb4ab62e Reviewed-on: https://go-review.googlesource.com/115296 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/net/http/httputil/reverseproxy_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/net/http/httputil/reverseproxy_test.go b/src/net/http/httputil/reverseproxy_test.go index 1ad67562af047..0240bfa8a616d 100644 --- a/src/net/http/httputil/reverseproxy_test.go +++ b/src/net/http/httputil/reverseproxy_test.go @@ -17,6 +17,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "reflect" "strconv" "strings" @@ -748,6 +749,8 @@ func TestServeHTTPDeepCopy(t *testing.T) { // Issue 18327: verify we always do a deep copy of the Request.Header map // before any mutations. func TestClonesRequestHeaders(t *testing.T) { + log.SetOutput(ioutil.Discard) + defer log.SetOutput(os.Stderr) req, _ := http.NewRequest("GET", "http://foo.tld/", nil) req.RemoteAddr = "1.2.3.4:56789" rp := &ReverseProxy{ @@ -824,6 +827,8 @@ func (cc *checkCloser) Read(b []byte) (int, error) { // Issue 23643: panic on body copy error func TestReverseProxy_PanicBodyError(t *testing.T) { + log.SetOutput(ioutil.Discard) + defer log.SetOutput(os.Stderr) backendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { out := "this call was relayed by the reverse proxy" // Coerce a wrong content length to induce io.ErrUnexpectedEOF From d4e21288e444d3ffd30d1a0737f15ea3fc3b8ad9 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 25 Apr 2018 11:06:41 -0400 Subject: [PATCH 041/187] cmd/go: add minimal module-awareness for legacy operation We want authors to be able to publish code that works with both the current standard go command and the planned new go command support for modules. If authors have tagged their code v2 or later, semantic import versioning means the import paths must include a v2 path element after the path prefix naming the module. One option for making this convention compatible with original go get is to move code into a v2 subdirectory of the root. That makes sense for some authors, but many authors would prefer not to move all the code into a v2 subdirectory for a transition and then move it back up once we everyone has a module-aware go command. Instead, this CL teaches the old (non-module-aware) go command a tiny amount about modules and their import paths, to expand the options for authors who want to publish compatible packages. If an author has a v2 of a package, say my/thing/v2/sub/pkg, in the my/thing repo's sub/pkg subdirectory (no v2 in the file system path), then old go get continues to import that package as my/thing/sub/pkg. But when go get is processing code in any module (code in a tree with a go.mod file) and encounters a path like my/thing/v2/sub/pkg, it will check to see if my/thing/go.mod says "module my/thing/v2". If so, the go command will read the import my/thing/v2/sub/pkg as if it said my/thing/sub/pkg, which is the correct "old" import path for the package in question. This CL will be back-ported to Go 1.10 and Go 1.9 as well. Once users have updated to the latest Go point releases containing this new logic, authors will be able to update to using modules within their own repos, including using semantic import paths with vN path elements, and old go get will still be able to consume those repositories. This CL also makes "go get" ignore meta go-import lines using the new "mod" VCS type. This allows a package to specify both a "mod" type and a "git" type, to present more efficient module access to module-aware go but still present a Git repo to the old "go get". Fixes #24751. Fixes #25069. Change-Id: I378955613a0d63834d4f50f121f4db7e4d87dc0a Reviewed-on: https://go-review.googlesource.com/109340 Run-TryBot: Russ Cox TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/get/discovery.go | 7 + src/cmd/go/internal/get/get.go | 2 +- src/cmd/go/internal/get/pkg_test.go | 14 + src/cmd/go/internal/list/list.go | 4 +- src/cmd/go/internal/load/pkg.go | 249 +++++++++++++++--- src/cmd/go/internal/load/test.go | 4 +- src/cmd/go/internal/test/test.go | 4 +- src/cmd/go/testdata/modlegacy/src/new/go.mod | 1 + src/cmd/go/testdata/modlegacy/src/new/new.go | 3 + .../go/testdata/modlegacy/src/new/p1/p1.go | 7 + .../go/testdata/modlegacy/src/new/p2/p2.go | 1 + .../go/testdata/modlegacy/src/new/sub/go.mod | 1 + .../modlegacy/src/new/sub/inner/go.mod | 1 + .../modlegacy/src/new/sub/inner/x/x.go | 1 + .../modlegacy/src/new/sub/x/v1/y/y.go | 1 + .../go/testdata/modlegacy/src/old/p1/p1.go | 5 + .../go/testdata/modlegacy/src/old/p2/p2.go | 1 + src/cmd/go/vendor_test.go | 31 +++ 18 files changed, 296 insertions(+), 41 deletions(-) create mode 100644 src/cmd/go/testdata/modlegacy/src/new/go.mod create mode 100644 src/cmd/go/testdata/modlegacy/src/new/new.go create mode 100644 src/cmd/go/testdata/modlegacy/src/new/p1/p1.go create mode 100644 src/cmd/go/testdata/modlegacy/src/new/p2/p2.go create mode 100644 src/cmd/go/testdata/modlegacy/src/new/sub/go.mod create mode 100644 src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod create mode 100644 src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go create mode 100644 src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go create mode 100644 src/cmd/go/testdata/modlegacy/src/old/p1/p1.go create mode 100644 src/cmd/go/testdata/modlegacy/src/old/p2/p2.go diff --git a/src/cmd/go/internal/get/discovery.go b/src/cmd/go/internal/get/discovery.go index b2918dbb4f31a..97aa1d7e8d634 100644 --- a/src/cmd/go/internal/get/discovery.go +++ b/src/cmd/go/internal/get/discovery.go @@ -55,6 +55,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { continue } if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { + // Ignore VCS type "mod", which is new Go modules. + // This code is for old go get and must ignore the new mod lines. + // Otherwise matchGoImport will complain about two + // different metaImport lines for the same Prefix. + if f[1] == "mod" { + continue + } imports = append(imports, metaImport{ Prefix: f[0], VCS: f[1], diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index 733116eca04d8..610c5407e9e24 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -209,7 +209,7 @@ var downloadRootCache = map[string]bool{} // download runs the download half of the get command // for the package named by the argument. func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) { - if mode&load.UseVendor != 0 { + if mode&load.ResolveImport != 0 { // Caller is responsible for expanding vendor paths. panic("internal error: download mode has useVendor set") } diff --git a/src/cmd/go/internal/get/pkg_test.go b/src/cmd/go/internal/get/pkg_test.go index b8937a57ec912..1179d86693ace 100644 --- a/src/cmd/go/internal/get/pkg_test.go +++ b/src/cmd/go/internal/get/pkg_test.go @@ -47,6 +47,20 @@ var parseMetaGoImportsTests = []struct { {"baz/quux", "git", "http://github.com/rsc/baz/quux"}, }, }, + { + ` + `, + []metaImport{ + {"foo/bar", "git", "https://github.com/rsc/foo/bar"}, + }, + }, + { + ` + `, + []metaImport{ + {"foo/bar", "git", "https://github.com/rsc/foo/bar"}, + }, + }, { ` diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 5b242a887a2d8..70bbf4bf3291b 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -296,8 +296,8 @@ func runList(cmd *base.Command, args []string) { for _, p := range pkgs { // Show vendor-expanded paths in listing - p.TestImports = p.Vendored(p.TestImports) - p.XTestImports = p.Vendored(p.XTestImports) + p.TestImports = p.Resolve(p.TestImports) + p.XTestImports = p.Resolve(p.XTestImports) } if *listTest { diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 5a26ca7892a23..dd8d18056cb13 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -6,6 +6,7 @@ package load import ( + "bytes" "fmt" "go/build" "go/token" @@ -14,6 +15,7 @@ import ( pathpkg "path" "path/filepath" "sort" + "strconv" "strings" "unicode" "unicode/utf8" @@ -179,7 +181,7 @@ func (e *NoGoError) Error() string { return "no Go files in " + e.Package.Dir } -// Vendored returns the vendor-resolved version of imports, +// Resolve returns the resolved version of imports, // which should be p.TestImports or p.XTestImports, NOT p.Imports. // The imports in p.TestImports and p.XTestImports are not recursively // loaded during the initial load of p, so they list the imports found in @@ -189,14 +191,14 @@ func (e *NoGoError) Error() string { // can produce better error messages if it starts with the original paths. // The initial load of p loads all the non-test imports and rewrites // the vendored paths, so nothing should ever call p.vendored(p.Imports). -func (p *Package) Vendored(imports []string) []string { +func (p *Package) Resolve(imports []string) []string { if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { - panic("internal error: p.vendored(p.Imports) called") + panic("internal error: p.Resolve(p.Imports) called") } seen := make(map[string]bool) var all []string for _, path := range imports { - path = VendoredImportPath(p, path) + path = ResolveImportPath(p, path) if !seen[path] { seen[path] = true all = append(all, path) @@ -391,16 +393,16 @@ func makeImportValid(r rune) rune { // Mode flags for loadImport and download (in get.go). const ( - // UseVendor means that loadImport should do vendor expansion - // (provided the vendoring experiment is enabled). - // That is, useVendor means that the import path came from - // a source file and has not been vendor-expanded yet. - // Every import path should be loaded initially with useVendor, - // and then the expanded version (with the /vendor/ in it) gets - // recorded as the canonical import path. At that point, future loads - // of that package must not pass useVendor, because + // ResolveImport means that loadImport should do import path expansion. + // That is, ResolveImport means that the import path came from + // a source file and has not been expanded yet to account for + // vendoring or possible module adjustment. + // Every import path should be loaded initially with ResolveImport, + // and then the expanded version (for example with the /vendor/ in it) + // gets recorded as the canonical import path. At that point, future loads + // of that package must not pass ResolveImport, because // disallowVendor will reject direct use of paths containing /vendor/. - UseVendor = 1 << iota + ResolveImport = 1 << iota // GetTestDeps is for download (part of "go get") and indicates // that test dependencies should be fetched too. @@ -425,12 +427,12 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo isLocal := build.IsLocalImport(path) if isLocal { importPath = dirToImportPath(filepath.Join(srcDir, path)) - } else if mode&UseVendor != 0 { - // We do our own vendor resolution, because we want to + } else if mode&ResolveImport != 0 { + // We do our own path resolution, because we want to // find out the key to use in packageCache without the // overhead of repeated calls to buildContext.Import. // The code is also needed in a few other places anyway. - path = VendoredImportPath(parent, path) + path = ResolveImportPath(parent, path) importPath = path } @@ -447,7 +449,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo // Import always returns bp != nil, even if an error occurs, // in order to return partial information. buildMode := build.ImportComment - if mode&UseVendor == 0 || path != origPath { + if mode&ResolveImport == 0 || path != origPath { // Not vendoring, or we already found the vendored path. buildMode |= build.IgnoreVendor } @@ -478,7 +480,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo if perr := disallowInternal(srcDir, p, stk); perr != p { return setErrorPos(perr, importPos) } - if mode&UseVendor != 0 { + if mode&ResolveImport != 0 { if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { return setErrorPos(perr, importPos) } @@ -537,24 +539,31 @@ func isDir(path string) bool { return result } -// VendoredImportPath returns the expansion of path when it appears in parent. -// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, -// x/vendor/path, vendor/path, or else stay path if none of those exist. -// VendoredImportPath returns the expanded path or, if no expansion is found, the original. -func VendoredImportPath(parent *Package, path string) (found string) { - if parent == nil || parent.Root == "" { - return path - } +// ResolveImportPath returns the true meaning of path when it appears in parent. +// There are two different resolutions applied. +// First, there is Go 1.5 vendoring (golang.org/s/go15vendor). +// If vendor expansion doesn't trigger, then the path is also subject to +// Go 1.11 vgo legacy conversion (golang.org/issue/25069). +func ResolveImportPath(parent *Package, path string) (found string) { + found = VendoredImportPath(parent, path) + if found != path { + return found + } + return ModuleImportPath(parent, path) +} - dir := filepath.Clean(parent.Dir) - root := filepath.Join(parent.Root, "src") - if !str.HasFilePathPrefix(dir, root) || parent.ImportPath != "command-line-arguments" && filepath.Join(root, parent.ImportPath) != dir { +// dirAndRoot returns the source directory and workspace root +// for the package p, guaranteeing that root is a path prefix of dir. +func dirAndRoot(p *Package) (dir, root string) { + dir = filepath.Clean(p.Dir) + root = filepath.Join(p.Root, "src") + if !str.HasFilePathPrefix(dir, root) || p.ImportPath != "command-line-arguments" && filepath.Join(root, p.ImportPath) != dir { // Look for symlinks before reporting error. dir = expandPath(dir) root = expandPath(root) } - if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || parent.ImportPath != "command-line-arguments" && !parent.Internal.Local && filepath.Join(root, parent.ImportPath) != dir { + if !str.HasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || p.ImportPath != "command-line-arguments" && !p.Internal.Local && filepath.Join(root, p.ImportPath) != dir { base.Fatalf("unexpected directory layout:\n"+ " import path: %s\n"+ " root: %s\n"+ @@ -562,14 +571,28 @@ func VendoredImportPath(parent *Package, path string) (found string) { " expand root: %s\n"+ " expand dir: %s\n"+ " separator: %s", - parent.ImportPath, - filepath.Join(parent.Root, "src"), - filepath.Clean(parent.Dir), + p.ImportPath, + filepath.Join(p.Root, "src"), + filepath.Clean(p.Dir), root, dir, string(filepath.Separator)) } + return dir, root +} + +// VendoredImportPath returns the vendor-expansion of path when it appears in parent. +// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, +// x/vendor/path, vendor/path, or else stay path if none of those exist. +// VendoredImportPath returns the expanded path or, if no expansion is found, the original. +func VendoredImportPath(parent *Package, path string) (found string) { + if parent == nil || parent.Root == "" { + return path + } + + dir, root := dirAndRoot(parent) + vpath := "vendor/" + path for i := len(dir); i >= len(root); i-- { if i < len(dir) && dir[i] != filepath.Separator { @@ -612,6 +635,164 @@ func VendoredImportPath(parent *Package, path string) (found string) { return path } +var ( + modulePrefix = []byte("\nmodule ") + goModPathCache = make(map[string]string) +) + +// goModPath returns the module path in the go.mod in dir, if any. +func goModPath(dir string) (path string) { + path, ok := goModPathCache[dir] + if ok { + return path + } + defer func() { + goModPathCache[dir] = path + }() + + data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + return "" + } + var i int + if bytes.HasPrefix(data, modulePrefix[1:]) { + i = 0 + } else { + i = bytes.Index(data, modulePrefix) + if i < 0 { + return "" + } + i++ + } + line := data[i:] + + // Cut line at \n, drop trailing \r if present. + if j := bytes.IndexByte(line, '\n'); j >= 0 { + line = line[:j] + } + if line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + line = line[len("module "):] + + // If quoted, unquote. + path = strings.TrimSpace(string(line)) + if path != "" && path[0] == '"' { + s, err := strconv.Unquote(path) + if err != nil { + return "" + } + path = s + } + return path +} + +// findVersionElement returns the slice indices of the final version element /vN in path. +// If there is no such element, it returns -1, -1. +func findVersionElement(path string) (i, j int) { + j = len(path) + for i = len(path) - 1; i >= 0; i-- { + if path[i] == '/' { + if isVersionElement(path[i:j]) { + return i, j + } + j = i + } + } + return -1, -1 +} + +// isVersionElement reports whether s is a well-formed path version element: +// v2, v3, v10, etc, but not v0, v05, v1. +func isVersionElement(s string) bool { + if len(s) < 3 || s[0] != '/' || s[1] != 'v' || s[2] == '0' || s[2] == '1' && len(s) == 3 { + return false + } + for i := 2; i < len(s); i++ { + if s[i] < '0' || '9' < s[i] { + return false + } + } + return true +} + +// ModuleImportPath translates import paths found in go modules +// back down to paths that can be resolved in ordinary builds. +// +// Define “new” code as code with a go.mod file in the same directory +// or a parent directory. If an import in new code says x/y/v2/z but +// x/y/v2/z does not exist and x/y/go.mod says “module x/y/v2”, +// then go build will read the import as x/y/z instead. +// See golang.org/issue/25069. +func ModuleImportPath(parent *Package, path string) (found string) { + if parent == nil || parent.Root == "" { + return path + } + + // If there are no vN elements in path, leave it alone. + // (The code below would do the same, but only after + // some other file system accesses that we can avoid + // here by returning early.) + if i, _ := findVersionElement(path); i < 0 { + return path + } + + dir, root := dirAndRoot(parent) + + // Consider dir and parents, up to and including root. + for i := len(dir); i >= len(root); i-- { + if i < len(dir) && dir[i] != filepath.Separator { + continue + } + if goModPath(dir[:i]) != "" { + goto HaveGoMod + } + } + // This code is not in a tree with a go.mod, + // so apply no changes to the path. + return path + +HaveGoMod: + // This import is in a tree with a go.mod. + // Allow it to refer to code in GOPATH/src/x/y/z as x/y/v2/z + // if GOPATH/src/x/y/go.mod says module "x/y/v2", + + // If x/y/v2/z exists, use it unmodified. + if bp, _ := cfg.BuildContext.Import(path, "", build.IgnoreVendor); bp.Dir != "" { + return path + } + + // Otherwise look for a go.mod supplying a version element. + // Some version-like elements may appear in paths but not + // be module versions; we skip over those to look for module + // versions. For example the module m/v2 might have a + // package m/v2/api/v1/foo. + limit := len(path) + for limit > 0 { + i, j := findVersionElement(path[:limit]) + if i < 0 { + return path + } + if bp, _ := cfg.BuildContext.Import(path[:i], "", build.IgnoreVendor); bp.Dir != "" { + if mpath := goModPath(bp.Dir); mpath != "" { + // Found a valid go.mod file, so we're stopping the search. + // If the path is m/v2/p and we found m/go.mod that says + // "module m/v2", then we return "m/p". + if mpath == path[:j] { + return path[:i] + path[j:] + } + // Otherwise just return the original path. + // We didn't find anything worth rewriting, + // and the go.mod indicates that we should + // not consider parent directories. + return path + } + } + limit = i + } + return path +} + // hasGoFiles reports whether dir contains any files with names ending in .go. // For a vendor check we must exclude directories that contain no .go files. // Otherwise it is not possible to vendor just a/b/c and still import the @@ -1076,7 +1257,7 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) { if path == "C" { continue } - p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], UseVendor) + p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil { p.Error = &PackageError{ ImportStack: stk.Copy(), diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index a9b47ce72dc40..1444ddb58a40f 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -58,7 +58,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], UseVendor) + p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error != nil { return nil, nil, nil, p1.Error } @@ -86,7 +86,7 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], UseVendor) + p1 := LoadImport(path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.Error != nil { return nil, nil, nil, p1.Error } diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index bcff5ff3b164d..585481b6b717d 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -600,10 +600,10 @@ func runTest(cmd *base.Command, args []string) { for _, path := range p.Imports { deps[path] = true } - for _, path := range p.Vendored(p.TestImports) { + for _, path := range p.Resolve(p.TestImports) { deps[path] = true } - for _, path := range p.Vendored(p.XTestImports) { + for _, path := range p.Resolve(p.XTestImports) { deps[path] = true } } diff --git a/src/cmd/go/testdata/modlegacy/src/new/go.mod b/src/cmd/go/testdata/modlegacy/src/new/go.mod new file mode 100644 index 0000000000000..d0dd46d314c6a --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/go.mod @@ -0,0 +1 @@ +module "new/v2" diff --git a/src/cmd/go/testdata/modlegacy/src/new/new.go b/src/cmd/go/testdata/modlegacy/src/new/new.go new file mode 100644 index 0000000000000..e99c47a6a8411 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/new.go @@ -0,0 +1,3 @@ +package new + +import _ "new/v2/p2" diff --git a/src/cmd/go/testdata/modlegacy/src/new/p1/p1.go b/src/cmd/go/testdata/modlegacy/src/new/p1/p1.go new file mode 100644 index 0000000000000..4539f40919653 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/p1/p1.go @@ -0,0 +1,7 @@ +package p1 + +import _ "old/p2" +import _ "new/v2" +import _ "new/v2/p2" +import _ "new/sub/v2/x/v1/y" // v2 is module, v1 is directory in module +import _ "new/sub/inner/x" // new/sub/inner/go.mod overrides new/sub/go.mod diff --git a/src/cmd/go/testdata/modlegacy/src/new/p2/p2.go b/src/cmd/go/testdata/modlegacy/src/new/p2/p2.go new file mode 100644 index 0000000000000..9b9052f54197e --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/p2/p2.go @@ -0,0 +1 @@ +package p2 diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/go.mod b/src/cmd/go/testdata/modlegacy/src/new/sub/go.mod new file mode 100644 index 0000000000000..484d20c6b2ea5 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/go.mod @@ -0,0 +1 @@ +module new/sub/v2 diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod new file mode 100644 index 0000000000000..ba3934541f745 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/go.mod @@ -0,0 +1 @@ +module new/sub/inner diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go new file mode 100644 index 0000000000000..823aafd0712b6 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/inner/x/x.go @@ -0,0 +1 @@ +package x diff --git a/src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go b/src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go new file mode 100644 index 0000000000000..789ca715ec460 --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/new/sub/x/v1/y/y.go @@ -0,0 +1 @@ +package y diff --git a/src/cmd/go/testdata/modlegacy/src/old/p1/p1.go b/src/cmd/go/testdata/modlegacy/src/old/p1/p1.go new file mode 100644 index 0000000000000..90527483abf2c --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/old/p1/p1.go @@ -0,0 +1,5 @@ +package p1 + +import _ "old/p2" +import _ "new/p1" +import _ "new" diff --git a/src/cmd/go/testdata/modlegacy/src/old/p2/p2.go b/src/cmd/go/testdata/modlegacy/src/old/p2/p2.go new file mode 100644 index 0000000000000..9b9052f54197e --- /dev/null +++ b/src/cmd/go/testdata/modlegacy/src/old/p2/p2.go @@ -0,0 +1 @@ +package p2 diff --git a/src/cmd/go/vendor_test.go b/src/cmd/go/vendor_test.go index 4f21a510a3cff..e30fc65d80457 100644 --- a/src/cmd/go/vendor_test.go +++ b/src/cmd/go/vendor_test.go @@ -328,3 +328,34 @@ func TestVendor12156(t *testing.T) { tg.grepStderrNot("panic", "panicked") tg.grepStderr(`cannot find package "x"`, "wrong error") } + +// Module legacy support does path rewriting very similar to vendoring. + +func TestModLegacy(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GOPATH", filepath.Join(tg.pwd(), "testdata/modlegacy")) + tg.run("list", "-f", "{{.Imports}}", "old/p1") + tg.grepStdout("new/p1", "old/p1 should import new/p1") + tg.run("list", "-f", "{{.Imports}}", "new/p1") + tg.grepStdout("new/p2", "new/p1 should import new/p2 (not new/v2/p2)") + tg.grepStdoutNot("new/v2", "new/p1 should NOT import new/v2*") + tg.grepStdout("new/sub/x/v1/y", "new/p1 should import new/sub/x/v1/y (not new/sub/v2/x/v1/y)") + tg.grepStdoutNot("new/sub/v2", "new/p1 should NOT import new/sub/v2*") + tg.grepStdout("new/sub/inner/x", "new/p1 should import new/sub/inner/x (no rewrites)") + tg.run("build", "old/p1", "new/p1") +} + +func TestModLegacyGet(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path(".")) + tg.run("get", "vcs-test.golang.org/git/modlegacy1-old.git/p1") + tg.run("list", "-f", "{{.Deps}}", "vcs-test.golang.org/git/modlegacy1-old.git/p1") + tg.grepStdout("new.git/p2", "old/p1 should depend on new/p2") + tg.grepStdoutNot("new.git/v2/p2", "old/p1 should NOT depend on new/v2/p2") + tg.run("build", "vcs-test.golang.org/git/modlegacy1-old.git/p1", "vcs-test.golang.org/git/modlegacy1-new.git/p1") +} From 0b8d9d425a66a3c7e1c76fe10cb9eab1acd316cc Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 25 May 2018 15:58:37 -0400 Subject: [PATCH 042/187] go/types: fix typo causing loss of embedded interfaces Simplified the code per prior suggestion to avoid that kind of error in the first place. Also: Fix subtle error in Interface.Complete where an interface may have ended up incomplete if both the list of methods and the list of embedded interfaces was nil. Expanded existing test to cover all these cases. Fixes golang/go#25577 Change-Id: If8723a8b0c4570f02b3dadfa390f96dd98ce11c8 Reviewed-on: https://go-review.googlesource.com/114504 Run-TryBot: Robert Griesemer Reviewed-by: Alan Donovan Reviewed-by: Robert Griesemer --- src/go/types/type.go | 46 ++++++++++++++------------------- src/go/types/typestring_test.go | 36 ++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/go/types/type.go b/src/go/types/type.go index cc87f1edb57d9..60e3efaec314e 100644 --- a/src/go/types/type.go +++ b/src/go/types/type.go @@ -264,12 +264,9 @@ var markComplete = make([]*Func, 0) // to be embedded. This is necessary for interfaces that embed alias type names referring to // non-defined (literal) interface types. func NewInterface(methods []*Func, embeddeds []*Named) *Interface { - var tnames []Type - if len(embeddeds) > 0 { - tnames := make([]Type, len(embeddeds)) - for i, t := range embeddeds { - tnames[i] = t - } + tnames := make([]Type, len(embeddeds)) + for i, t := range embeddeds { + tnames[i] = t } return NewInterface2(methods, tnames) } @@ -356,27 +353,24 @@ func (t *Interface) Complete() *Interface { } var allMethods []*Func - if t.embeddeds == nil { - if t.methods == nil { - allMethods = make([]*Func, 0, 1) - } else { - allMethods = t.methods - } - } else { - allMethods = append(allMethods, t.methods...) - for _, et := range t.embeddeds { - it := et.Underlying().(*Interface) - it.Complete() - for _, tm := range it.allMethods { - // Make a copy of the method and adjust its receiver type. - newm := *tm - newmtyp := *tm.typ.(*Signature) - newm.typ = &newmtyp - newmtyp.recv = NewVar(newm.pos, newm.pkg, "", t) - allMethods = append(allMethods, &newm) - } + allMethods = append(allMethods, t.methods...) + for _, et := range t.embeddeds { + it := et.Underlying().(*Interface) + it.Complete() + for _, tm := range it.allMethods { + // Make a copy of the method and adjust its receiver type. + newm := *tm + newmtyp := *tm.typ.(*Signature) + newm.typ = &newmtyp + newmtyp.recv = NewVar(newm.pos, newm.pkg, "", t) + allMethods = append(allMethods, &newm) } - sort.Sort(byUniqueMethodName(allMethods)) + } + sort.Sort(byUniqueMethodName(allMethods)) + + // t.methods and/or t.embeddeds may have been empty + if allMethods == nil { + allMethods = markComplete } t.allMethods = allMethods diff --git a/src/go/types/typestring_test.go b/src/go/types/typestring_test.go index 78f67d1f05002..6ed2d75dfe3f1 100644 --- a/src/go/types/typestring_test.go +++ b/src/go/types/typestring_test.go @@ -140,16 +140,41 @@ func TestTypeString(t *testing.T) { func TestIncompleteInterfaces(t *testing.T) { sig := NewSignature(nil, nil, nil, false) + m := NewFunc(token.NoPos, nil, "m", sig) for _, test := range []struct { typ *Interface want string }{ {new(Interface), "interface{/* incomplete */}"}, {new(Interface).Complete(), "interface{}"}, + + {NewInterface(nil, nil), "interface{/* incomplete */}"}, + {NewInterface(nil, nil).Complete(), "interface{}"}, + {NewInterface([]*Func{}, nil), "interface{/* incomplete */}"}, + {NewInterface([]*Func{}, nil).Complete(), "interface{}"}, + {NewInterface(nil, []*Named{}), "interface{/* incomplete */}"}, + {NewInterface(nil, []*Named{}).Complete(), "interface{}"}, + {NewInterface([]*Func{m}, nil), "interface{m() /* incomplete */}"}, + {NewInterface([]*Func{m}, nil).Complete(), "interface{m()}"}, + {NewInterface(nil, []*Named{newDefined(new(Interface).Complete())}), "interface{T /* incomplete */}"}, + {NewInterface(nil, []*Named{newDefined(new(Interface).Complete())}).Complete(), "interface{T}"}, + {NewInterface(nil, []*Named{newDefined(NewInterface([]*Func{m}, nil))}), "interface{T /* incomplete */}"}, + {NewInterface(nil, []*Named{newDefined(NewInterface([]*Func{m}, nil).Complete())}), "interface{T /* incomplete */}"}, + {NewInterface(nil, []*Named{newDefined(NewInterface([]*Func{m}, nil).Complete())}).Complete(), "interface{T}"}, + {NewInterface2(nil, nil), "interface{/* incomplete */}"}, {NewInterface2(nil, nil).Complete(), "interface{}"}, - {NewInterface2([]*Func{NewFunc(token.NoPos, nil, "m", sig)}, nil), "interface{m() /* incomplete */}"}, - {NewInterface2([]*Func{NewFunc(token.NoPos, nil, "m", sig)}, nil).Complete(), "interface{m()}"}, + {NewInterface2([]*Func{}, nil), "interface{/* incomplete */}"}, + {NewInterface2([]*Func{}, nil).Complete(), "interface{}"}, + {NewInterface2(nil, []Type{}), "interface{/* incomplete */}"}, + {NewInterface2(nil, []Type{}).Complete(), "interface{}"}, + {NewInterface2([]*Func{m}, nil), "interface{m() /* incomplete */}"}, + {NewInterface2([]*Func{m}, nil).Complete(), "interface{m()}"}, + {NewInterface2(nil, []Type{new(Interface).Complete()}), "interface{interface{} /* incomplete */}"}, + {NewInterface2(nil, []Type{new(Interface).Complete()}).Complete(), "interface{interface{}}"}, + {NewInterface2(nil, []Type{NewInterface2([]*Func{m}, nil)}), "interface{interface{m() /* incomplete */} /* incomplete */}"}, + {NewInterface2(nil, []Type{NewInterface2([]*Func{m}, nil).Complete()}), "interface{interface{m()} /* incomplete */}"}, + {NewInterface2(nil, []Type{NewInterface2([]*Func{m}, nil).Complete()}).Complete(), "interface{interface{m()}}"}, } { got := test.typ.String() if got != test.want { @@ -158,6 +183,13 @@ func TestIncompleteInterfaces(t *testing.T) { } } +// newDefined creates a new defined type named T with the given underlying type. +// Helper function for use with TestIncompleteInterfaces only. +func newDefined(underlying Type) *Named { + tname := NewTypeName(token.NoPos, nil, "T", nil) + return NewNamed(tname, underlying, nil) +} + func TestQualifiedTypeString(t *testing.T) { p, _ := pkgFor("p.go", "package p; type T int", nil) q, _ := pkgFor("q.go", "package q", nil) From 9b49edef50a86ac06d956774bc03a2410f73920f Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 30 May 2018 20:07:41 +0100 Subject: [PATCH 043/187] net: fix leftover variable names from CL 115175 Change-Id: I5f78fe3286bf4667b6922c57c5701c09bf56e182 Reviewed-on: https://go-review.googlesource.com/115355 Reviewed-by: Brad Fitzpatrick --- src/net/unixsock.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/net/unixsock.go b/src/net/unixsock.go index bd7dc398487f9..06beaecc28b63 100644 --- a/src/net/unixsock.go +++ b/src/net/unixsock.go @@ -317,8 +317,8 @@ func ListenUnix(network string, laddr *UnixAddr) (*UnixListener, error) { if laddr == nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} } - sa := &sysListener{network: network, address: laddr.String()} - ln, err := sa.listenUnix(context.Background(), laddr) + sl := &sysListener{network: network, address: laddr.String()} + ln, err := sl.listenUnix(context.Background(), laddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } @@ -337,8 +337,8 @@ func ListenUnixgram(network string, laddr *UnixAddr) (*UnixConn, error) { if laddr == nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: errMissingAddress} } - sa := &sysListener{network: network, address: laddr.String()} - c, err := sa.listenUnixgram(context.Background(), laddr) + sl := &sysListener{network: network, address: laddr.String()} + c, err := sl.listenUnixgram(context.Background(), laddr) if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } From 323c85862a7afbde66a3bba0776bf4ba6cd7c030 Mon Sep 17 00:00:00 2001 From: Ilya Tocar Date: Tue, 15 May 2018 13:29:18 -0500 Subject: [PATCH 044/187] strconv: simplify (*extFloat).Normalize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use math/bits.LeadingZeros64 instead of local implementation. This simplifies code, makes Normalize inlinable and fixes performance regression. Idea was suggested by Giovanni Bajo in #25298 Performance results below: Atof64Decimal-6 46.7ns ± 0% 46.7ns ± 0% ~ (all equal) Atof64Float-6 57.9ns ± 1% 56.9ns ± 0% -1.72% (p=0.000 n=10+9) Atof64FloatExp-6 163ns ± 0% 123ns ± 0% -24.54% (p=0.002 n=8+10) Atof64Big-6 222ns ± 1% 185ns ± 1% -16.65% (p=0.000 n=9+10) Atof64RandomBits-6 155ns ± 2% 154ns ± 3% ~ (p=0.225 n=10+10) Atof64RandomFloats-6 156ns ± 2% 154ns ± 2% ~ (p=0.124 n=10+9) Atof32Decimal-6 47.3ns ± 0% 46.7ns ± 0% -1.26% (p=0.000 n=7+9) Atof32Float-6 51.5ns ± 1% 51.6ns ± 1% ~ (p=0.455 n=10+9) Atof32FloatExp-6 163ns ± 1% 124ns ± 1% -24.36% (p=0.000 n=10+10) Atof32Random-6 199ns ± 1% 163ns ± 0% -17.93% (p=0.000 n=10+10) FormatFloat/Decimal-6 209ns ± 2% 211ns ± 2% ~ (p=0.402 n=10+10) FormatFloat/Float-6 393ns ± 2% 379ns ± 1% -3.57% (p=0.000 n=10+10) FormatFloat/Exp-6 333ns ± 2% 321ns ± 1% -3.56% (p=0.000 n=10+9) FormatFloat/NegExp-6 338ns ± 3% 317ns ± 1% -6.27% (p=0.000 n=10+9) FormatFloat/Big-6 457ns ± 1% 443ns ± 2% -2.99% (p=0.000 n=9+10) FormatFloat/BinaryExp-6 230ns ± 2% 232ns ± 2% ~ (p=0.070 n=10+10) FormatFloat/32Integer-6 209ns ± 2% 211ns ± 1% ~ (p=0.203 n=10+8) FormatFloat/32ExactFraction-6 330ns ± 2% 319ns ± 1% -3.42% (p=0.000 n=10+10) FormatFloat/32Point-6 393ns ± 2% 377ns ± 1% -4.15% (p=0.000 n=10+10) FormatFloat/32Exp-6 331ns ± 2% 318ns ± 2% -4.02% (p=0.000 n=10+10) FormatFloat/32NegExp-6 327ns ± 2% 315ns ± 2% -3.70% (p=0.000 n=10+10) FormatFloat/64Fixed1-6 265ns ± 2% 253ns ± 2% -4.38% (p=0.000 n=10+10) FormatFloat/64Fixed2-6 278ns ± 2% 262ns ± 3% -5.71% (p=0.000 n=10+10) FormatFloat/64Fixed3-6 271ns ± 2% 260ns ± 2% -4.03% (p=0.000 n=10+10) FormatFloat/64Fixed4-6 277ns ± 3% 267ns ± 1% -3.55% (p=0.000 n=10+9) FormatFloat/Slowpath64-6 71.0µs ± 0% 71.0µs ± 0% ~ (p=0.744 n=10+8) AppendFloat/Decimal-6 100ns ± 1% 100ns ± 0% ~ (p=0.294 n=10+8) AppendFloat/Float-6 273ns ± 0% 260ns ± 1% -4.87% (p=0.000 n=7+10) AppendFloat/Exp-6 213ns ± 0% 200ns ± 0% -6.29% (p=0.000 n=8+10) AppendFloat/NegExp-6 211ns ± 0% 198ns ± 0% -6.16% (p=0.000 n=8+8) AppendFloat/Big-6 319ns ± 0% 305ns ± 0% -4.31% (p=0.000 n=8+7) AppendFloat/BinaryExp-6 98.4ns ± 0% 92.9ns ± 0% -5.63% (p=0.000 n=9+8) AppendFloat/32Integer-6 101ns ± 1% 102ns ± 1% +0.89% (p=0.004 n=10+10) AppendFloat/32ExactFraction-6 222ns ± 1% 210ns ± 0% -5.28% (p=0.000 n=10+9) AppendFloat/32Point-6 273ns ± 1% 261ns ± 1% -4.62% (p=0.000 n=10+9) AppendFloat/32Exp-6 209ns ± 1% 197ns ± 0% -5.56% (p=0.000 n=10+9) AppendFloat/32NegExp-6 207ns ± 1% 194ns ± 1% -6.18% (p=0.000 n=10+10) AppendFloat/64Fixed1-6 145ns ± 0% 131ns ± 1% -9.93% (p=0.000 n=9+10) AppendFloat/64Fixed2-6 160ns ± 0% 146ns ± 0% -8.58% (p=0.000 n=10+8) AppendFloat/64Fixed3-6 147ns ± 1% 132ns ± 1% -10.25% (p=0.000 n=10+10) AppendFloat/64Fixed4-6 161ns ± 1% 149ns ± 0% -7.93% (p=0.000 n=10+10) AppendFloat/Slowpath64-6 70.6µs ± 1% 70.9µs ± 0% +0.37% (p=0.000 n=10+8) Change-Id: I63bbc40905abd795fbd24743604c790023d11a43 Reviewed-on: https://go-review.googlesource.com/113256 Run-TryBot: Ilya Tocar TryBot-Result: Gobot Gobot Reviewed-by: Robert Griesemer --- src/strconv/extfloat.go | 41 +++++++++++------------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/src/strconv/extfloat.go b/src/strconv/extfloat.go index 7f17bc6a0df36..558daa1dbe6cc 100644 --- a/src/strconv/extfloat.go +++ b/src/strconv/extfloat.go @@ -4,6 +4,10 @@ package strconv +import ( + "math/bits" +) + // An extFloat represents an extended floating-point number, with more // precision than a float64. It does not try to save bits: the // number represented by the structure is mant*(2^exp), with a negative @@ -196,38 +200,15 @@ func (f *extFloat) AssignComputeBounds(mant uint64, exp int, neg bool, flt *floa // Normalize normalizes f so that the highest bit of the mantissa is // set, and returns the number by which the mantissa was left-shifted. -func (f *extFloat) Normalize() (shift uint) { - mant, exp := f.mant, f.exp - if mant == 0 { +func (f *extFloat) Normalize() uint { + // bits.LeadingZeros64 would return 64 + if f.mant == 0 { return 0 } - if mant>>(64-32) == 0 { - mant <<= 32 - exp -= 32 - } - if mant>>(64-16) == 0 { - mant <<= 16 - exp -= 16 - } - if mant>>(64-8) == 0 { - mant <<= 8 - exp -= 8 - } - if mant>>(64-4) == 0 { - mant <<= 4 - exp -= 4 - } - if mant>>(64-2) == 0 { - mant <<= 2 - exp -= 2 - } - if mant>>(64-1) == 0 { - mant <<= 1 - exp -= 1 - } - shift = uint(f.exp - exp) - f.mant, f.exp = mant, exp - return + shift := bits.LeadingZeros64(f.mant) + f.mant <<= uint(shift) + f.exp -= shift + return uint(shift) } // Multiply sets f to the product f*g: the result is correctly rounded, From 7ea2c8cf1b8052b98532fe95c91fc685758bc249 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 30 May 2018 21:54:36 +0200 Subject: [PATCH 045/187] os/exec: gofmt CL 109361 introduced some changes which were not properly gofmt'ed. Because the CL was sent via Github no gofmt checks were performed on it (cf. #24946, #18548). Change-Id: I207065f01161044c420e272f4fd112e0a59be259 Reviewed-on: https://go-review.googlesource.com/115356 Run-TryBot: Tobias Klauser Reviewed-by: Brad Fitzpatrick --- src/os/exec/lp_windows_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/exec/lp_windows_test.go b/src/os/exec/lp_windows_test.go index 64d7dca2e85f1..59b5f1c2c7ced 100644 --- a/src/os/exec/lp_windows_test.go +++ b/src/os/exec/lp_windows_test.go @@ -117,7 +117,7 @@ func createEnv(dir, PATH, PATHEXT string) []string { dirs[i] = filepath.Join(dir, dirs[i]) } path := strings.Join(dirs, ";") - env = updateEnv(env, "PATH", os.Getenv("SystemRoot") + "/System32;" + path) + env = updateEnv(env, "PATH", os.Getenv("SystemRoot")+"/System32;"+path) return env } From 999e230ac17390f8e1dd917db1769c447b10585b Mon Sep 17 00:00:00 2001 From: Mikio Hara Date: Sat, 28 Apr 2018 12:27:53 +0900 Subject: [PATCH 046/187] net/http: use DialWithConn method of socks.Dialer This change uses the DialWithConn method of socks.Dialer to ensure that the bundled SOCKS client implementation is agnostic to the behavior and capabilities of transport connections. Also updates the bundled golang.org/x/net/internal/socks at git rev 7594486. (golang.org/cl/110135) Updates #25104. Change-Id: I87c2e99eeb857f182ea5d8ef569181d4f45f2e5d Reviewed-on: https://go-review.googlesource.com/110136 Run-TryBot: Mikio Hara TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/net/http/socks_bundle.go | 79 +++++++++++++++++++++++++++++------- src/net/http/transport.go | 5 +-- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/net/http/socks_bundle.go b/src/net/http/socks_bundle.go index 8b347898e8501..e4314b4128306 100644 --- a/src/net/http/socks_bundle.go +++ b/src/net/http/socks_bundle.go @@ -1,5 +1,5 @@ // Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. -//go:generate bundle -o socks_bundle.go -dst http -prefix socks -underscore golang.org/x/net/internal/socks +//go:generate bundle -o socks_bundle.go -dst net/http -prefix socks -underscore golang.org/x/net/internal/socks // Package socks provides a SOCKS version 5 client implementation. // @@ -305,20 +305,13 @@ type socksDialer struct { // See func Dial of the net package of standard library for a // description of the network and address parameters. func (d *socksDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - switch network { - case "tcp", "tcp6", "tcp4": - default: + if err := d.validateTarget(network, address); err != nil { proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("network not implemented")} - } - switch d.cmd { - case socksCmdConnect, sockscmdBind: - default: - proxy, dst, _ := d.pathAddrs(address) - return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("command not implemented")} + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} } if ctx == nil { - ctx = context.Background() + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} } var err error var c net.Conn @@ -341,11 +334,69 @@ func (d *socksDialer) DialContext(ctx context.Context, network, address string) return &socksConn{Conn: c, boundAddr: a}, nil } +// DialWithConn initiates a connection from SOCKS server to the target +// network and address using the connection c that is already +// connected to the SOCKS server. +// +// It returns the connection's local address assigned by the SOCKS +// server. +func (d *socksDialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) { + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if ctx == nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} + } + a, err := d.connect(ctx, c, address) + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + return a, nil +} + // Dial connects to the provided address on the provided network. // -// Deprecated: Use DialContext instead. +// Unlike DialContext, it returns a raw transport connection instead +// of a forward proxy connection. +// +// Deprecated: Use DialContext or DialWithConn instead. func (d *socksDialer) Dial(network, address string) (net.Conn, error) { - return d.DialContext(context.Background(), network, address) + if err := d.validateTarget(network, address); err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + var err error + var c net.Conn + if d.ProxyDial != nil { + c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress) + } else { + c, err = net.Dial(d.proxyNetwork, d.proxyAddress) + } + if err != nil { + proxy, dst, _ := d.pathAddrs(address) + return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err} + } + if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil { + return nil, err + } + return c, nil +} + +func (d *socksDialer) validateTarget(network, address string) error { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return errors.New("network not implemented") + } + switch d.cmd { + case socksCmdConnect, sockscmdBind: + default: + return errors.New("command not implemented") + } + return nil } func (d *socksDialer) pathAddrs(address string) (proxy, dst net.Addr, err error) { diff --git a/src/net/http/transport.go b/src/net/http/transport.go index 731bf176a8723..ba2c00e067b7f 100644 --- a/src/net/http/transport.go +++ b/src/net/http/transport.go @@ -1086,9 +1086,6 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistCon case cm.proxyURL.Scheme == "socks5": conn := pconn.conn d := socksNewDialer("tcp", conn.RemoteAddr().String()) - d.ProxyDial = func(_ context.Context, _, _ string) (net.Conn, error) { - return conn, nil - } if u := cm.proxyURL.User; u != nil { auth := &socksUsernamePassword{ Username: u.Username(), @@ -1100,7 +1097,7 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistCon } d.Authenticate = auth.Authenticate } - if _, err := d.DialContext(ctx, "tcp", cm.targetAddr); err != nil { + if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil { conn.Close() return nil, err } From cc6e568c818053ddc16b80b0406a87d19de7a120 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 30 May 2018 15:07:04 -0700 Subject: [PATCH 047/187] cmd/go: accept more safe CFLAGS/LDFLAGS Fixes #23749 Fixes #24703 Fixes #24858 Change-Id: Ib32d8efee294004c70fdd602087df2da0867f099 Reviewed-on: https://go-review.googlesource.com/115415 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/cmd/go/internal/work/security.go | 37 ++++++++++++++++++++--- src/cmd/go/internal/work/security_test.go | 5 +++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/cmd/go/internal/work/security.go b/src/cmd/go/internal/work/security.go index 880f4fdc791a3..cd39a8f79181f 100644 --- a/src/cmd/go/internal/work/security.go +++ b/src/cmd/go/internal/work/security.go @@ -41,43 +41,57 @@ var re = regexp.MustCompile var validCompilerFlags = []*regexp.Regexp{ re(`-D([A-Za-z_].*)`), + re(`-F([^@\-].*)`), re(`-I([^@\-].*)`), re(`-O`), re(`-O([^@\-].*)`), re(`-W`), re(`-W([^@,]+)`), // -Wall but not -Wa,-foo. re(`-Wa,-mbig-obj`), + re(`-Wp,-D([A-Za-z_].*)`), re(`-ansi`), + re(`-f(no-)?asynchronous-unwind-tables`), re(`-f(no-)?blocks`), + re(`-f(no-)builtin-[a-zA-Z0-9_]*`), re(`-f(no-)?common`), re(`-f(no-)?constant-cfstrings`), re(`-fdiagnostics-show-note-include-stack`), + re(`-f(no-)?eliminate-unused-debug-types`), re(`-f(no-)?exceptions`), + re(`-f(no-)?fast-math`), re(`-f(no-)?inline-functions`), re(`-finput-charset=([^@\-].*)`), re(`-f(no-)?fat-lto-objects`), + re(`-f(no-)?keep-inline-dllexport`), re(`-f(no-)?lto`), re(`-fmacro-backtrace-limit=(.+)`), re(`-fmessage-length=(.+)`), re(`-f(no-)?modules`), re(`-f(no-)?objc-arc`), + re(`-f(no-)?objc-nonfragile-abi`), + re(`-f(no-)?objc-legacy-dispatch`), re(`-f(no-)?omit-frame-pointer`), re(`-f(no-)?openmp(-simd)?`), re(`-f(no-)?permissive`), re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?plt`), re(`-f(no-)?rtti`), re(`-f(no-)?split-stack`), re(`-f(no-)?stack-(.+)`), re(`-f(no-)?strict-aliasing`), re(`-f(un)signed-char`), re(`-f(no-)?use-linker-plugin`), // safe if -B is not used; we don't permit -B + re(`-f(no-)?visibility-inlines-hidden`), re(`-fsanitize=(.+)`), re(`-ftemplate-depth-(.+)`), re(`-fvisibility=(.+)`), re(`-g([^@\-].*)?`), re(`-m32`), re(`-m64`), - re(`-m(arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-marm`), + re(`-mfloat-abi=([^@\-].*)`), + re(`-mfpmath=[0-9a-z,+]*`), re(`-m(no-)?avx[0-9a-z.]*`), re(`-m(no-)?ms-bitfields`), re(`-m(no-)?stack-(.+)`), @@ -86,12 +100,16 @@ var validCompilerFlags = []*regexp.Regexp{ re(`-miphoneos-version-min=(.+)`), re(`-mnop-fun-dllimport`), re(`-m(no-)?sse[0-9.]*`), + re(`-mthumb(-interwork)?`), + re(`-mthreads`), re(`-mwindows`), + re(`--param=ssp-buffer-size=[0-9]*`), re(`-pedantic(-errors)?`), re(`-pipe`), re(`-pthread`), re(`-?-std=([^@\-].*)`), re(`-?-stdlib=([^@\-].*)`), + re(`--sysroot=([^@\-].*)`), re(`-w`), re(`-x([^@\-].*)`), re(`-v`), @@ -116,15 +134,20 @@ var validLinkerFlags = []*regexp.Regexp{ re(`-O`), re(`-O([^@\-].*)`), re(`-f(no-)?(pic|PIC|pie|PIE)`), + re(`-f(no-)?openmp(-simd)?`), re(`-fsanitize=([^@\-].*)`), re(`-g([^@\-].*)?`), - re(`-m(arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-headerpad_max_install_names`), + re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`), + re(`-mfloat-abi=([^@\-].*)`), re(`-mmacosx-(.+)`), re(`-mios-simulator-version-min=(.+)`), re(`-miphoneos-version-min=(.+)`), + re(`-mthreads`), re(`-mwindows`), re(`-(pic|PIC|pie|PIE)`), re(`-pthread`), + re(`-rdynamic`), re(`-shared`), re(`-?-static([-a-z0-9+]*)`), re(`-?-stdlib=([^@\-].*)`), @@ -136,22 +159,27 @@ var validLinkerFlags = []*regexp.Regexp{ // in a wildcard would allow tunnelling arbitrary additional // linker arguments through one of these. re(`-Wl,--(no-)?allow-multiple-definition`), + re(`-Wl,--(no-)?allow-shlib-undefined`), re(`-Wl,--(no-)?as-needed`), re(`-Wl,-Bdynamic`), re(`-Wl,-Bstatic`), + re(`-WL,-O([^@,\-][^,]*)?`), re(`-Wl,-d[ny]`), re(`-Wl,--disable-new-dtags`), + re(`-Wl,-e[=,][a-zA-Z0-9]*`), re(`-Wl,--enable-new-dtags`), re(`-Wl,--end-group`), re(`-Wl,-framework,[^,@\-][^,]+`), re(`-Wl,-headerpad_max_install_names`), re(`-Wl,--no-undefined`), - re(`-Wl,-rpath[=,]([^,@\-][^,]+)`), + re(`-Wl,-rpath(-link)?[=,]([^,@\-][^,]+)`), + re(`-Wl,-s`), re(`-Wl,-search_paths_first`), re(`-Wl,-sectcreate,([^,@\-][^,]+),([^,@\-][^,]+),([^,@\-][^,]+)`), re(`-Wl,--start-group`), re(`-Wl,-?-static`), - re(`-Wl,--subsystem,(native|windows|console|posix|xbox)`), + re(`-Wl,-?-subsystem,(native|windows|console|posix|xbox)`), + re(`-Wl,-syslibroot[=,]([^,@\-][^,]+)`), re(`-Wl,-undefined[=,]([^,@\-][^,]+)`), re(`-Wl,-?-unresolved-symbols=[^,]+`), re(`-Wl,--(no-)?warn-([^,]+)`), @@ -159,6 +187,7 @@ var validLinkerFlags = []*regexp.Regexp{ re(`-Wl,-z,relro`), re(`[a-zA-Z0-9_/].*\.(a|o|obj|dll|dylib|so)`), // direct linker inputs: x.o or libfoo.so (but not -foo.o or @foo.o) + re(`\./.*\.(a|o|obj|dll|dylib|so)`), } var validLinkerFlagsWithNextArg = []string{ diff --git a/src/cmd/go/internal/work/security_test.go b/src/cmd/go/internal/work/security_test.go index 15eeff9b4b4bc..d23b6eadff50e 100644 --- a/src/cmd/go/internal/work/security_test.go +++ b/src/cmd/go/internal/work/security_test.go @@ -12,6 +12,7 @@ import ( var goodCompilerFlags = [][]string{ {"-DFOO"}, {"-Dfoo=bar"}, + {"-F/Qt"}, {"-I/"}, {"-I/etc/passwd"}, {"-I."}, @@ -63,6 +64,8 @@ var goodCompilerFlags = [][]string{ var badCompilerFlags = [][]string{ {"-D@X"}, {"-D-X"}, + {"-F@dir"}, + {"-F-dir"}, {"-I@dir"}, {"-I-dir"}, {"-O@1"}, @@ -126,6 +129,7 @@ var goodLinkerFlags = [][]string{ {"-Wl,--no-warn-error"}, {"foo.so"}, {"_世界.dll"}, + {"./x.o"}, {"libcgosotest.dylib"}, {"-F", "framework"}, {"-l", "."}, @@ -193,6 +197,7 @@ var badLinkerFlags = [][]string{ {"-x", "--c"}, {"-x", "@obj"}, {"-Wl,-rpath,@foo"}, + {"../x.o"}, } func TestCheckLinkerFlags(t *testing.T) { From 3c4d3bdd3b454ef45ce00559d705fe5dc6f57cad Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Mon, 28 May 2018 02:47:21 +0100 Subject: [PATCH 048/187] net: add ListenConfig, Dialer.Control to permit socket opts before listen/dial Existing implementation does not provide a way to set options such as SO_REUSEPORT, that has to be set prior the socket being bound. New exposed API: pkg net, method (*ListenConfig) Listen(context.Context, string, string) (Listener, error) pkg net, method (*ListenConfig) ListenPacket(context.Context, string, string) (PacketConn, error) pkg net, type ListenConfig struct pkg net, type ListenConfig struct, Control func(string, string, syscall.RawConn) error pkg net, type Dialer struct, Control func(string, string, syscall.RawConn) error Fixes #9661 Change-Id: If4d275711f823df72d3ac5cc3858651a6a57cccb Reviewed-on: https://go-review.googlesource.com/72810 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/net/dial.go | 123 ++++++++++++++++++++++---------- src/net/dial_test.go | 51 +++++++++++++ src/net/iprawsock_posix.go | 4 +- src/net/ipsock_posix.go | 4 +- src/net/listen_test.go | 54 ++++++++++++++ src/net/rawconn_stub_test.go | 4 ++ src/net/rawconn_unix_test.go | 38 +++++++++- src/net/rawconn_windows_test.go | 30 ++++++++ src/net/sock_posix.go | 94 ++++++++++++++++++------ src/net/tcpsock_posix.go | 6 +- src/net/udpsock_posix.go | 6 +- src/net/unixsock_posix.go | 10 +-- 12 files changed, 350 insertions(+), 74 deletions(-) diff --git a/src/net/dial.go b/src/net/dial.go index 3ea049ca460e7..b1a5ca7cd53f5 100644 --- a/src/net/dial.go +++ b/src/net/dial.go @@ -8,6 +8,7 @@ import ( "context" "internal/nettrace" "internal/poll" + "syscall" "time" ) @@ -70,6 +71,14 @@ type Dialer struct { // // Deprecated: Use DialContext instead. Cancel <-chan struct{} + + // If Control is not nil, it is called after creating the network + // connection but before actually dialing. + // + // Network and address parameters passed to Control method are not + // necessarily the ones passed to Dial. For example, passing "tcp" to Dial + // will cause the Control function to be called with "tcp4" or "tcp6". + Control func(network, address string, c syscall.RawConn) error } func minNonzeroTime(a, b time.Time) time.Time { @@ -563,8 +572,82 @@ func (sd *sysDialer) dialSingle(ctx context.Context, ra Addr) (c Conn, err error return c, nil } +// ListenConfig contains options for listening to an address. +type ListenConfig struct { + // If Control is not nil, it is called after creating the network + // connection but before binding it to the operating system. + // + // Network and address parameters passed to Control method are not + // necessarily the ones passed to Listen. For example, passing "tcp" to + // Listen will cause the Control function to be called with "tcp4" or "tcp6". + Control func(network, address string, c syscall.RawConn) error +} + +// Listen announces on the local network address. +// +// See func Listen for a description of the network and address +// parameters. +func (lc *ListenConfig) Listen(ctx context.Context, network, address string) (Listener, error) { + addrs, err := DefaultResolver.resolveAddrList(ctx, "listen", network, address, nil) + if err != nil { + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: err} + } + sl := &sysListener{ + ListenConfig: *lc, + network: network, + address: address, + } + var l Listener + la := addrs.first(isIPv4) + switch la := la.(type) { + case *TCPAddr: + l, err = sl.listenTCP(ctx, la) + case *UnixAddr: + l, err = sl.listenUnix(ctx, la) + default: + return nil, &OpError{Op: "listen", Net: sl.network, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: address}} + } + if err != nil { + return nil, &OpError{Op: "listen", Net: sl.network, Source: nil, Addr: la, Err: err} // l is non-nil interface containing nil pointer + } + return l, nil +} + +// ListenPacket announces on the local network address. +// +// See func ListenPacket for a description of the network and address +// parameters. +func (lc *ListenConfig) ListenPacket(ctx context.Context, network, address string) (PacketConn, error) { + addrs, err := DefaultResolver.resolveAddrList(ctx, "listen", network, address, nil) + if err != nil { + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: err} + } + sl := &sysListener{ + ListenConfig: *lc, + network: network, + address: address, + } + var c PacketConn + la := addrs.first(isIPv4) + switch la := la.(type) { + case *UDPAddr: + c, err = sl.listenUDP(ctx, la) + case *IPAddr: + c, err = sl.listenIP(ctx, la) + case *UnixAddr: + c, err = sl.listenUnixgram(ctx, la) + default: + return nil, &OpError{Op: "listen", Net: sl.network, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: address}} + } + if err != nil { + return nil, &OpError{Op: "listen", Net: sl.network, Source: nil, Addr: la, Err: err} // c is non-nil interface containing nil pointer + } + return c, nil +} + // sysListener contains a Listen's parameters and configuration. type sysListener struct { + ListenConfig network, address string } @@ -587,23 +670,8 @@ type sysListener struct { // See func Dial for a description of the network and address // parameters. func Listen(network, address string) (Listener, error) { - addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", network, address, nil) - if err != nil { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: err} - } - var l Listener - switch la := addrs.first(isIPv4).(type) { - case *TCPAddr: - l, err = ListenTCP(network, la) - case *UnixAddr: - l, err = ListenUnix(network, la) - default: - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: address}} - } - if err != nil { - return nil, err // l is non-nil interface containing nil pointer - } - return l, nil + var lc ListenConfig + return lc.Listen(context.Background(), network, address) } // ListenPacket announces on the local network address. @@ -629,23 +697,6 @@ func Listen(network, address string) (Listener, error) { // See func Dial for a description of the network and address // parameters. func ListenPacket(network, address string) (PacketConn, error) { - addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", network, address, nil) - if err != nil { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: nil, Err: err} - } - var l PacketConn - switch la := addrs.first(isIPv4).(type) { - case *UDPAddr: - l, err = ListenUDP(network, la) - case *IPAddr: - l, err = ListenIP(network, la) - case *UnixAddr: - l, err = ListenUnixgram(network, la) - default: - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: address}} - } - if err != nil { - return nil, err // l is non-nil interface containing nil pointer - } - return l, nil + var lc ListenConfig + return lc.ListenPacket(context.Background(), network, address) } diff --git a/src/net/dial_test.go b/src/net/dial_test.go index 96d8921ec8525..3934ad8648360 100644 --- a/src/net/dial_test.go +++ b/src/net/dial_test.go @@ -912,6 +912,57 @@ func TestDialListenerAddr(t *testing.T) { c.Close() } +func TestDialerControl(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9": + t.Skipf("not supported on %s", runtime.GOOS) + } + + t.Run("StreamDial", func(t *testing.T) { + for _, network := range []string{"tcp", "tcp4", "tcp6", "unix", "unixpacket"} { + if !testableNetwork(network) { + continue + } + ln, err := newLocalListener(network) + if err != nil { + t.Error(err) + continue + } + defer ln.Close() + d := Dialer{Control: controlOnConnSetup} + c, err := d.Dial(network, ln.Addr().String()) + if err != nil { + t.Error(err) + continue + } + c.Close() + } + }) + t.Run("PacketDial", func(t *testing.T) { + for _, network := range []string{"udp", "udp4", "udp6", "unixgram"} { + if !testableNetwork(network) { + continue + } + c1, err := newLocalPacketListener(network) + if err != nil { + t.Error(err) + continue + } + if network == "unixgram" { + defer os.Remove(c1.LocalAddr().String()) + } + defer c1.Close() + d := Dialer{Control: controlOnConnSetup} + c2, err := d.Dial(network, c1.LocalAddr().String()) + if err != nil { + t.Error(err) + continue + } + c2.Close() + } + }) +} + // mustHaveExternalNetwork is like testenv.MustHaveExternalNetwork // except that it won't skip testing on non-iOS builders. func mustHaveExternalNetwork(t *testing.T) { diff --git a/src/net/iprawsock_posix.go b/src/net/iprawsock_posix.go index 7dafd20bf6861..b2f57916433eb 100644 --- a/src/net/iprawsock_posix.go +++ b/src/net/iprawsock_posix.go @@ -122,7 +122,7 @@ func (sd *sysDialer) dialIP(ctx context.Context, laddr, raddr *IPAddr) (*IPConn, default: return nil, UnknownNetworkError(sd.network) } - fd, err := internetSocket(ctx, network, laddr, raddr, syscall.SOCK_RAW, proto, "dial") + fd, err := internetSocket(ctx, network, laddr, raddr, syscall.SOCK_RAW, proto, "dial", sd.Dialer.Control) if err != nil { return nil, err } @@ -139,7 +139,7 @@ func (sl *sysListener) listenIP(ctx context.Context, laddr *IPAddr) (*IPConn, er default: return nil, UnknownNetworkError(sl.network) } - fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_RAW, proto, "listen") + fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_RAW, proto, "listen", sl.ListenConfig.Control) if err != nil { return nil, err } diff --git a/src/net/ipsock_posix.go b/src/net/ipsock_posix.go index 8372aaa7423cf..eddd4118fa231 100644 --- a/src/net/ipsock_posix.go +++ b/src/net/ipsock_posix.go @@ -133,12 +133,12 @@ func favoriteAddrFamily(network string, laddr, raddr sockaddr, mode string) (fam return syscall.AF_INET6, false } -func internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string) (fd *netFD, err error) { +func internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string, ctrlFn func(string, string, syscall.RawConn) error) (fd *netFD, err error) { if (runtime.GOOS == "windows" || runtime.GOOS == "openbsd" || runtime.GOOS == "nacl") && mode == "dial" && raddr.isWildcard() { raddr = raddr.toLocal(net) } family, ipv6only := favoriteAddrFamily(net, laddr, raddr, mode) - return socket(ctx, net, family, sotype, proto, ipv6only, laddr, raddr) + return socket(ctx, net, family, sotype, proto, ipv6only, laddr, raddr, ctrlFn) } func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, error) { diff --git a/src/net/listen_test.go b/src/net/listen_test.go index 96624f98ce53f..ffd38d79506d6 100644 --- a/src/net/listen_test.go +++ b/src/net/listen_test.go @@ -7,6 +7,7 @@ package net import ( + "context" "fmt" "internal/testenv" "os" @@ -729,3 +730,56 @@ func TestClosingListener(t *testing.T) { } ln2.Close() } + +func TestListenConfigControl(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9": + t.Skipf("not supported on %s", runtime.GOOS) + } + + t.Run("StreamListen", func(t *testing.T) { + for _, network := range []string{"tcp", "tcp4", "tcp6", "unix", "unixpacket"} { + if !testableNetwork(network) { + continue + } + ln, err := newLocalListener(network) + if err != nil { + t.Error(err) + continue + } + address := ln.Addr().String() + ln.Close() + lc := ListenConfig{Control: controlOnConnSetup} + ln, err = lc.Listen(context.Background(), network, address) + if err != nil { + t.Error(err) + continue + } + ln.Close() + } + }) + t.Run("PacketListen", func(t *testing.T) { + for _, network := range []string{"udp", "udp4", "udp6", "unixgram"} { + if !testableNetwork(network) { + continue + } + c, err := newLocalPacketListener(network) + if err != nil { + t.Error(err) + continue + } + address := c.LocalAddr().String() + c.Close() + if network == "unixgram" { + os.Remove(address) + } + lc := ListenConfig{Control: controlOnConnSetup} + c, err = lc.ListenPacket(context.Background(), network, address) + if err != nil { + t.Error(err) + continue + } + c.Close() + } + }) +} diff --git a/src/net/rawconn_stub_test.go b/src/net/rawconn_stub_test.go index 391b4d188e2e2..3e3b6bf5b2a1a 100644 --- a/src/net/rawconn_stub_test.go +++ b/src/net/rawconn_stub_test.go @@ -22,3 +22,7 @@ func writeRawConn(c syscall.RawConn, b []byte) error { func controlRawConn(c syscall.RawConn, addr Addr) error { return errors.New("not supported") } + +func controlOnConnSetup(network string, address string, c syscall.RawConn) error { + return nil +} diff --git a/src/net/rawconn_unix_test.go b/src/net/rawconn_unix_test.go index 2fe4d2c6bace6..a720a8a4a3e27 100644 --- a/src/net/rawconn_unix_test.go +++ b/src/net/rawconn_unix_test.go @@ -6,7 +6,10 @@ package net -import "syscall" +import ( + "errors" + "syscall" +) func readRawConn(c syscall.RawConn, b []byte) (int, error) { var operr error @@ -89,3 +92,36 @@ func controlRawConn(c syscall.RawConn, addr Addr) error { } return nil } + +func controlOnConnSetup(network string, address string, c syscall.RawConn) error { + var operr error + var fn func(uintptr) + switch network { + case "tcp", "udp", "ip": + return errors.New("ambiguous network: " + network) + case "unix", "unixpacket", "unixgram": + fn = func(s uintptr) { + _, operr = syscall.GetsockoptInt(int(s), syscall.SOL_SOCKET, syscall.SO_ERROR) + } + default: + switch network[len(network)-1] { + case '4': + fn = func(s uintptr) { + operr = syscall.SetsockoptInt(int(s), syscall.IPPROTO_IP, syscall.IP_TTL, 1) + } + case '6': + fn = func(s uintptr) { + operr = syscall.SetsockoptInt(int(s), syscall.IPPROTO_IPV6, syscall.IPV6_UNICAST_HOPS, 1) + } + default: + return errors.New("unknown network: " + network) + } + } + if err := c.Control(fn); err != nil { + return err + } + if operr != nil { + return operr + } + return nil +} diff --git a/src/net/rawconn_windows_test.go b/src/net/rawconn_windows_test.go index 6df101e9de4b2..2774c97e5c82e 100644 --- a/src/net/rawconn_windows_test.go +++ b/src/net/rawconn_windows_test.go @@ -5,6 +5,7 @@ package net import ( + "errors" "syscall" "unsafe" ) @@ -96,3 +97,32 @@ func controlRawConn(c syscall.RawConn, addr Addr) error { } return nil } + +func controlOnConnSetup(network string, address string, c syscall.RawConn) error { + var operr error + var fn func(uintptr) + switch network { + case "tcp", "udp", "ip": + return errors.New("ambiguous network: " + network) + default: + switch network[len(network)-1] { + case '4': + fn = func(s uintptr) { + operr = syscall.SetsockoptInt(syscall.Handle(s), syscall.IPPROTO_IP, syscall.IP_TTL, 1) + } + case '6': + fn = func(s uintptr) { + operr = syscall.SetsockoptInt(syscall.Handle(s), syscall.IPPROTO_IPV6, syscall.IPV6_UNICAST_HOPS, 1) + } + default: + return errors.New("unknown network: " + network) + } + } + if err := c.Control(fn); err != nil { + return err + } + if operr != nil { + return operr + } + return nil +} diff --git a/src/net/sock_posix.go b/src/net/sock_posix.go index 8cfc42eb7e66f..00ff3fd39394c 100644 --- a/src/net/sock_posix.go +++ b/src/net/sock_posix.go @@ -38,7 +38,7 @@ type sockaddr interface { // socket returns a network file descriptor that is ready for // asynchronous I/O using the network poller. -func socket(ctx context.Context, net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr) (fd *netFD, err error) { +func socket(ctx context.Context, net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr, ctrlFn func(string, string, syscall.RawConn) error) (fd *netFD, err error) { s, err := sysSocket(family, sotype, proto) if err != nil { return nil, err @@ -77,26 +77,41 @@ func socket(ctx context.Context, net string, family, sotype, proto int, ipv6only if laddr != nil && raddr == nil { switch sotype { case syscall.SOCK_STREAM, syscall.SOCK_SEQPACKET: - if err := fd.listenStream(laddr, listenerBacklog); err != nil { + if err := fd.listenStream(laddr, listenerBacklog, ctrlFn); err != nil { fd.Close() return nil, err } return fd, nil case syscall.SOCK_DGRAM: - if err := fd.listenDatagram(laddr); err != nil { + if err := fd.listenDatagram(laddr, ctrlFn); err != nil { fd.Close() return nil, err } return fd, nil } } - if err := fd.dial(ctx, laddr, raddr); err != nil { + if err := fd.dial(ctx, laddr, raddr, ctrlFn); err != nil { fd.Close() return nil, err } return fd, nil } +func (fd *netFD) ctrlNetwork() string { + switch fd.net { + case "unix", "unixgram", "unixpacket": + return fd.net + } + switch fd.net[len(fd.net)-1] { + case '4', '6': + return fd.net + } + if fd.family == syscall.AF_INET { + return fd.net + "4" + } + return fd.net + "6" +} + func (fd *netFD) addrFunc() func(syscall.Sockaddr) Addr { switch fd.family { case syscall.AF_INET, syscall.AF_INET6: @@ -121,14 +136,29 @@ func (fd *netFD) addrFunc() func(syscall.Sockaddr) Addr { return func(syscall.Sockaddr) Addr { return nil } } -func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr) error { +func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr, ctrlFn func(string, string, syscall.RawConn) error) error { + if ctrlFn != nil { + c, err := newRawConn(fd) + if err != nil { + return err + } + var ctrlAddr string + if raddr != nil { + ctrlAddr = raddr.String() + } else if laddr != nil { + ctrlAddr = laddr.String() + } + if err := ctrlFn(fd.ctrlNetwork(), ctrlAddr, c); err != nil { + return err + } + } var err error var lsa syscall.Sockaddr if laddr != nil { if lsa, err = laddr.sockaddr(fd.family); err != nil { return err } else if lsa != nil { - if err := syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { + if err = syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { return os.NewSyscallError("bind", err) } } @@ -165,29 +195,39 @@ func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr) error { return nil } -func (fd *netFD) listenStream(laddr sockaddr, backlog int) error { - if err := setDefaultListenerSockopts(fd.pfd.Sysfd); err != nil { +func (fd *netFD) listenStream(laddr sockaddr, backlog int, ctrlFn func(string, string, syscall.RawConn) error) error { + var err error + if err = setDefaultListenerSockopts(fd.pfd.Sysfd); err != nil { return err } - if lsa, err := laddr.sockaddr(fd.family); err != nil { + var lsa syscall.Sockaddr + if lsa, err = laddr.sockaddr(fd.family); err != nil { return err - } else if lsa != nil { - if err := syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { - return os.NewSyscallError("bind", err) + } + if ctrlFn != nil { + c, err := newRawConn(fd) + if err != nil { + return err + } + if err := ctrlFn(fd.ctrlNetwork(), laddr.String(), c); err != nil { + return err } } - if err := listenFunc(fd.pfd.Sysfd, backlog); err != nil { + if err = syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { + return os.NewSyscallError("bind", err) + } + if err = listenFunc(fd.pfd.Sysfd, backlog); err != nil { return os.NewSyscallError("listen", err) } - if err := fd.init(); err != nil { + if err = fd.init(); err != nil { return err } - lsa, _ := syscall.Getsockname(fd.pfd.Sysfd) + lsa, _ = syscall.Getsockname(fd.pfd.Sysfd) fd.setAddr(fd.addrFunc()(lsa), nil) return nil } -func (fd *netFD) listenDatagram(laddr sockaddr) error { +func (fd *netFD) listenDatagram(laddr sockaddr, ctrlFn func(string, string, syscall.RawConn) error) error { switch addr := laddr.(type) { case *UDPAddr: // We provide a socket that listens to a wildcard @@ -211,17 +251,27 @@ func (fd *netFD) listenDatagram(laddr sockaddr) error { laddr = &addr } } - if lsa, err := laddr.sockaddr(fd.family); err != nil { + var err error + var lsa syscall.Sockaddr + if lsa, err = laddr.sockaddr(fd.family); err != nil { return err - } else if lsa != nil { - if err := syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { - return os.NewSyscallError("bind", err) + } + if ctrlFn != nil { + c, err := newRawConn(fd) + if err != nil { + return err + } + if err := ctrlFn(fd.ctrlNetwork(), laddr.String(), c); err != nil { + return err } } - if err := fd.init(); err != nil { + if err = syscall.Bind(fd.pfd.Sysfd, lsa); err != nil { + return os.NewSyscallError("bind", err) + } + if err = fd.init(); err != nil { return err } - lsa, _ := syscall.Getsockname(fd.pfd.Sysfd) + lsa, _ = syscall.Getsockname(fd.pfd.Sysfd) fd.setAddr(fd.addrFunc()(lsa), nil) return nil } diff --git a/src/net/tcpsock_posix.go b/src/net/tcpsock_posix.go index 6061c16986c22..bcf7592d35f79 100644 --- a/src/net/tcpsock_posix.go +++ b/src/net/tcpsock_posix.go @@ -62,7 +62,7 @@ func (sd *sysDialer) dialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPCo } func (sd *sysDialer) doDialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConn, error) { - fd, err := internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") + fd, err := internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, "dial", sd.Dialer.Control) // TCP has a rarely used mechanism called a 'simultaneous connection' in // which Dial("tcp", addr1, addr2) run on the machine at addr1 can @@ -92,7 +92,7 @@ func (sd *sysDialer) doDialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCP if err == nil { fd.Close() } - fd, err = internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") + fd, err = internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, "dial", sd.Dialer.Control) } if err != nil { @@ -156,7 +156,7 @@ func (ln *TCPListener) file() (*os.File, error) { } func (sl *sysListener) listenTCP(ctx context.Context, laddr *TCPAddr) (*TCPListener, error) { - fd, err := internetSocket(ctx, sl.network, laddr, nil, syscall.SOCK_STREAM, 0, "listen") + fd, err := internetSocket(ctx, sl.network, laddr, nil, syscall.SOCK_STREAM, 0, "listen", sl.ListenConfig.Control) if err != nil { return nil, err } diff --git a/src/net/udpsock_posix.go b/src/net/udpsock_posix.go index 4e96f4781df47..8f4b71c01ec80 100644 --- a/src/net/udpsock_posix.go +++ b/src/net/udpsock_posix.go @@ -95,7 +95,7 @@ func (c *UDPConn) writeMsg(b, oob []byte, addr *UDPAddr) (n, oobn int, err error } func (sd *sysDialer) dialUDP(ctx context.Context, laddr, raddr *UDPAddr) (*UDPConn, error) { - fd, err := internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_DGRAM, 0, "dial") + fd, err := internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_DGRAM, 0, "dial", sd.Dialer.Control) if err != nil { return nil, err } @@ -103,7 +103,7 @@ func (sd *sysDialer) dialUDP(ctx context.Context, laddr, raddr *UDPAddr) (*UDPCo } func (sl *sysListener) listenUDP(ctx context.Context, laddr *UDPAddr) (*UDPConn, error) { - fd, err := internetSocket(ctx, sl.network, laddr, nil, syscall.SOCK_DGRAM, 0, "listen") + fd, err := internetSocket(ctx, sl.network, laddr, nil, syscall.SOCK_DGRAM, 0, "listen", sl.ListenConfig.Control) if err != nil { return nil, err } @@ -111,7 +111,7 @@ func (sl *sysListener) listenUDP(ctx context.Context, laddr *UDPAddr) (*UDPConn, } func (sl *sysListener) listenMulticastUDP(ctx context.Context, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - fd, err := internetSocket(ctx, sl.network, gaddr, nil, syscall.SOCK_DGRAM, 0, "listen") + fd, err := internetSocket(ctx, sl.network, gaddr, nil, syscall.SOCK_DGRAM, 0, "listen", sl.ListenConfig.Control) if err != nil { return nil, err } diff --git a/src/net/unixsock_posix.go b/src/net/unixsock_posix.go index f627567af5f0f..2495da1d253fa 100644 --- a/src/net/unixsock_posix.go +++ b/src/net/unixsock_posix.go @@ -13,7 +13,7 @@ import ( "syscall" ) -func unixSocket(ctx context.Context, net string, laddr, raddr sockaddr, mode string) (*netFD, error) { +func unixSocket(ctx context.Context, net string, laddr, raddr sockaddr, mode string, ctrlFn func(string, string, syscall.RawConn) error) (*netFD, error) { var sotype int switch net { case "unix": @@ -42,7 +42,7 @@ func unixSocket(ctx context.Context, net string, laddr, raddr sockaddr, mode str return nil, errors.New("unknown mode: " + mode) } - fd, err := socket(ctx, net, syscall.AF_UNIX, sotype, 0, false, laddr, raddr) + fd, err := socket(ctx, net, syscall.AF_UNIX, sotype, 0, false, laddr, raddr, ctrlFn) if err != nil { return nil, err } @@ -151,7 +151,7 @@ func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err err } func (sd *sysDialer) dialUnix(ctx context.Context, laddr, raddr *UnixAddr) (*UnixConn, error) { - fd, err := unixSocket(ctx, sd.network, laddr, raddr, "dial") + fd, err := unixSocket(ctx, sd.network, laddr, raddr, "dial", sd.Dialer.Control) if err != nil { return nil, err } @@ -207,7 +207,7 @@ func (l *UnixListener) SetUnlinkOnClose(unlink bool) { } func (sl *sysListener) listenUnix(ctx context.Context, laddr *UnixAddr) (*UnixListener, error) { - fd, err := unixSocket(ctx, sl.network, laddr, nil, "listen") + fd, err := unixSocket(ctx, sl.network, laddr, nil, "listen", sl.ListenConfig.Control) if err != nil { return nil, err } @@ -215,7 +215,7 @@ func (sl *sysListener) listenUnix(ctx context.Context, laddr *UnixAddr) (*UnixLi } func (sl *sysListener) listenUnixgram(ctx context.Context, laddr *UnixAddr) (*UnixConn, error) { - fd, err := unixSocket(ctx, sl.network, laddr, nil, "listen") + fd, err := unixSocket(ctx, sl.network, laddr, nil, "listen", sl.ListenConfig.Control) if err != nil { return nil, err } From fffb3a5c20b31a8d6916697ebac19ac9e8d3f6e7 Mon Sep 17 00:00:00 2001 From: Yury Smolsky Date: Tue, 15 May 2018 09:55:46 +0300 Subject: [PATCH 049/187] testing: make indentation consistent in sub-tests Instead of mixed usage of spaces and tabs for indentation, just use 4 spaces instead of one tab. This test: func TestX(t *testing.T) { t.Error("1\nnew line") t.Error("2") t.Error("3") t.Run("Y", func(t *testing.T) { t.Error("2") t.Error("2b\nnew line") t.Run("Z", func(t *testing.T) { t.Error("3\nnew line") }) }) t.Error("4") } produces following output: --- FAIL: TestX (0.00s) indent_test.go:6: 1 new line indent_test.go:7: 2 indent_test.go:8: 3 --- FAIL: TestX/Y (0.00s) indent_test.go:10: 2 indent_test.go:11: 2b new line --- FAIL: TestX/Y/Z (0.00s) indent_test.go:13: 3 new line indent_test.go:16: 4 FAIL Fixes #25369 Change-Id: Ib3b5da45ab3ee670c6e8a23172e7cbefb94c5e60 Reviewed-on: https://go-review.googlesource.com/113177 Run-TryBot: Yury Smolsky TryBot-Result: Gobot Gobot Reviewed-by: Marcel van Lohuizen --- src/cmd/go/go_test.go | 32 ++++++++++++++++---------------- src/testing/sub_test.go | 32 ++++++++++++++++---------------- src/testing/testing.go | 10 +++++----- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index a0fc72aac4af2..c32be94823086 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -4892,30 +4892,30 @@ func TestTestRegexps(t *testing.T) { // BenchmarkX/Y is run in full, twice want := `=== RUN TestX === RUN TestX/Y - x_test.go:6: LOG: X running - x_test.go:8: LOG: Y running + x_test.go:6: LOG: X running + x_test.go:8: LOG: Y running === RUN TestXX - z_test.go:10: LOG: XX running + z_test.go:10: LOG: XX running === RUN TestX === RUN TestX/Y - x_test.go:6: LOG: X running - x_test.go:8: LOG: Y running + x_test.go:6: LOG: X running + x_test.go:8: LOG: Y running === RUN TestXX - z_test.go:10: LOG: XX running + z_test.go:10: LOG: XX running --- BENCH: BenchmarkX/Y - x_test.go:15: LOG: Y running N=1 - x_test.go:15: LOG: Y running N=100 - x_test.go:15: LOG: Y running N=10000 - x_test.go:15: LOG: Y running N=1000000 - x_test.go:15: LOG: Y running N=100000000 - x_test.go:15: LOG: Y running N=2000000000 + x_test.go:15: LOG: Y running N=1 + x_test.go:15: LOG: Y running N=100 + x_test.go:15: LOG: Y running N=10000 + x_test.go:15: LOG: Y running N=1000000 + x_test.go:15: LOG: Y running N=100000000 + x_test.go:15: LOG: Y running N=2000000000 --- BENCH: BenchmarkX/Y - x_test.go:15: LOG: Y running N=1 - x_test.go:15: LOG: Y running N=2000000000 + x_test.go:15: LOG: Y running N=1 + x_test.go:15: LOG: Y running N=2000000000 --- BENCH: BenchmarkX - x_test.go:13: LOG: X running N=1 + x_test.go:13: LOG: X running N=1 --- BENCH: BenchmarkXX - z_test.go:18: LOG: XX running N=1 + z_test.go:18: LOG: XX running N=1 ` have := strings.Join(lines, "") diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index a5e6a1fb418ed..9af3909b3556e 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -168,7 +168,7 @@ func TestTRun(t *T) { --- FAIL: failure in parallel test propagates upwards (N.NNs) --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs) --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs) - `, + `, f: func(t *T) { t.Run("", func(t *T) { t.Parallel() @@ -210,8 +210,8 @@ func TestTRun(t *T) { desc: "skipping after error", output: ` --- FAIL: skipping after error (N.NNs) - sub_test.go:NNN: an error - sub_test.go:NNN: skipped`, + sub_test.go:NNN: an error + sub_test.go:NNN: skipped`, f: func(t *T) { t.Error("an error") t.Skip("skipped") @@ -320,9 +320,9 @@ func TestTRun(t *T) { ok: false, output: ` --- FAIL: subtest calls error on parent (N.NNs) - sub_test.go:NNN: first this - sub_test.go:NNN: and now this! - sub_test.go:NNN: oh, and this too`, + sub_test.go:NNN: first this + sub_test.go:NNN: and now this! + sub_test.go:NNN: oh, and this too`, maxPar: 1, f: func(t *T) { t.Errorf("first this") @@ -337,10 +337,10 @@ func TestTRun(t *T) { ok: false, output: ` --- FAIL: subtest calls fatal on parent (N.NNs) - sub_test.go:NNN: first this - sub_test.go:NNN: and now this! + sub_test.go:NNN: first this + sub_test.go:NNN: and now this! --- FAIL: subtest calls fatal on parent/#00 (N.NNs) - testing.go:NNN: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test`, + testing.go:NNN: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test`, maxPar: 1, f: func(t *T) { outer := t @@ -355,10 +355,10 @@ func TestTRun(t *T) { ok: false, output: ` --- FAIL: subtest calls error on ancestor (N.NNs) - sub_test.go:NNN: Report to ancestor + sub_test.go:NNN: Report to ancestor --- FAIL: subtest calls error on ancestor/#00 (N.NNs) - sub_test.go:NNN: Still do this - sub_test.go:NNN: Also do this`, + sub_test.go:NNN: Still do this + sub_test.go:NNN: Also do this`, maxPar: 1, f: func(t *T) { outer := t @@ -375,7 +375,7 @@ func TestTRun(t *T) { ok: false, output: ` --- FAIL: subtest calls fatal on ancestor (N.NNs) - sub_test.go:NNN: Nope`, + sub_test.go:NNN: Nope`, maxPar: 1, f: func(t *T) { outer := t @@ -503,7 +503,7 @@ func TestBRun(t *T) { chatty: true, output: ` --- SKIP: root - sub_test.go:NNN: skipping`, + sub_test.go:NNN: skipping`, f: func(b *B) { b.Skip("skipping") }, }, { desc: "chatty with recursion", @@ -521,8 +521,8 @@ func TestBRun(t *T) { failed: true, output: ` --- FAIL: root - sub_test.go:NNN: an error - sub_test.go:NNN: skipped`, + sub_test.go:NNN: an error + sub_test.go:NNN: skipped`, f: func(b *B) { b.Error("an error") b.Skip("skipped") diff --git a/src/testing/testing.go b/src/testing/testing.go index 6865645444026..a552b363617c6 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -394,7 +394,7 @@ func (c *common) frameSkip(skip int) runtime.Frame { } // decorate prefixes the string with the file and line of the call site -// and inserts the final newline if needed and indentation tabs for formatting. +// and inserts the final newline if needed and indentation spaces for formatting. // This function must be called with c.mu held. func (c *common) decorate(s string) string { frame := c.frameSkip(3) // decorate + log + public function. @@ -414,8 +414,8 @@ func (c *common) decorate(s string) string { line = 1 } buf := new(strings.Builder) - // Every line is indented at least one tab. - buf.WriteByte('\t') + // Every line is indented at least 4 spaces. + buf.WriteString(" ") fmt.Fprintf(buf, "%s:%d: ", file, line) lines := strings.Split(s, "\n") if l := len(lines); l > 1 && lines[l-1] == "" { @@ -423,8 +423,8 @@ func (c *common) decorate(s string) string { } for i, line := range lines { if i > 0 { - // Second and subsequent lines are indented an extra tab. - buf.WriteString("\n\t\t") + // Second and subsequent lines are indented an additional 4 spaces. + buf.WriteString("\n ") } buf.WriteString(line) } From caf968616a3ee09d6c820428b4edebb68cfbde09 Mon Sep 17 00:00:00 2001 From: Yury Smolsky Date: Wed, 30 May 2018 19:46:59 +0300 Subject: [PATCH 050/187] test: eliminate use of Perl in fixedbugs/bug345.go To allow testing of fixedbugs/bug345.go in Go, a new flag -n is introduced. This flag disables setting of relative path for local imports and imports search path to current dir, namely -D . -I . are not passed to the compiler. Error regexps are fixed to allow running the test in temp directory. This change eliminates the last place where Perl script "errchk" was used. Fixes #25586. Change-Id: If085f466e6955312d77315f96d3ef1cb68495aef Reviewed-on: https://go-review.googlesource.com/115277 Run-TryBot: Yury Smolsky TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- test/fixedbugs/bug345.dir/main.go | 7 ++--- test/fixedbugs/bug345.go | 45 +++---------------------------- test/run.go | 21 +++++++++++---- 3 files changed, 23 insertions(+), 50 deletions(-) diff --git a/test/fixedbugs/bug345.dir/main.go b/test/fixedbugs/bug345.dir/main.go index 6e4fdf4e21ab5..b77a2fad5fba0 100644 --- a/test/fixedbugs/bug345.dir/main.go +++ b/test/fixedbugs/bug345.dir/main.go @@ -6,8 +6,9 @@ package main import ( "bufio" - "./io" goio "io" + + "./io" ) func main() { @@ -22,7 +23,7 @@ func main() { // main.go:27: cannot use &x (type *"io".SectionReader) as type *"/Users/rsc/g/go/test/fixedbugs/bug345.dir/io".SectionReader in function argument var w io.Writer - bufio.NewWriter(w) // ERROR "test/io|has incompatible type" + bufio.NewWriter(w) // ERROR "[\w.]+[^.]/io|has incompatible type" var x goio.SectionReader - io.SR(&x) // ERROR "test/io|has incompatible type" + io.SR(&x) // ERROR "[\w.]+[^.]/io|has incompatible type" } diff --git a/test/fixedbugs/bug345.go b/test/fixedbugs/bug345.go index af505c8a3bea1..917592118dee0 100644 --- a/test/fixedbugs/bug345.go +++ b/test/fixedbugs/bug345.go @@ -1,47 +1,8 @@ -// +build !nacl,!js,!plan9,!windows -// run +// errorcheckdir -n +// run // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "regexp" -) - -func main() { - // TODO: If we get rid of errchk, re-enable this test on Plan 9 and Windows. - errchk, err := filepath.Abs("errchk") - check(err) - - bugDir := filepath.Join(".", "fixedbugs", "bug345.dir") - run("go", "tool", "compile", filepath.Join(bugDir, "io.go")) - run(errchk, "go", "tool", "compile", "-e", filepath.Join(bugDir, "main.go")) - - os.Remove("io.o") -} - -var bugRE = regexp.MustCompile(`(?m)^BUG`) - -func run(name string, args ...string) { - cmd := exec.Command(name, args...) - out, err := cmd.CombinedOutput() - if bugRE.Match(out) || err != nil { - fmt.Println(string(out)) - fmt.Println(err) - os.Exit(1) - } -} - -func check(err error) { - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} +package ignored diff --git a/test/run.go b/test/run.go index 81c0c0b92950e..0805ecd4fcfe8 100644 --- a/test/run.go +++ b/test/run.go @@ -216,8 +216,12 @@ func compileFile(runcmd runCmd, longname string, flags []string) (out []byte, er return runcmd(cmd...) } -func compileInDir(runcmd runCmd, dir string, flags []string, names ...string) (out []byte, err error) { - cmd := []string{goTool(), "tool", "compile", "-e", "-D", ".", "-I", "."} +func compileInDir(runcmd runCmd, dir string, flags []string, localImports bool, names ...string) (out []byte, err error) { + cmd := []string{goTool(), "tool", "compile", "-e"} + if localImports { + // Set relative path for local imports and import search path to current dir. + cmd = append(cmd, "-D", ".", "-I", ".") + } cmd = append(cmd, flags...) if *linkshared { cmd = append(cmd, "-dynlink", "-installsuffix=dynlink") @@ -489,6 +493,7 @@ func (t *test) run() { wantError := false wantAuto := false singlefilepkgs := false + localImports := true f := strings.Fields(action) if len(f) > 0 { action = f[0] @@ -530,6 +535,12 @@ func (t *test) run() { wantError = false case "-s": singlefilepkgs = true + case "-n": + // Do not set relative path for local imports to current dir, + // e.g. do not pass -D . -I . to the compiler. + // Used in fixedbugs/bug345.go to allow compilation and import of local pkg. + // See golang.org/issue/25635 + localImports = false case "-t": // timeout in seconds args = args[1:] var err error @@ -668,7 +679,7 @@ func (t *test) run() { return } for _, gofiles := range pkgs { - _, t.err = compileInDir(runcmd, longdir, flags, gofiles...) + _, t.err = compileInDir(runcmd, longdir, flags, localImports, gofiles...) if t.err != nil { return } @@ -690,7 +701,7 @@ func (t *test) run() { errPkg-- } for i, gofiles := range pkgs { - out, err := compileInDir(runcmd, longdir, flags, gofiles...) + out, err := compileInDir(runcmd, longdir, flags, localImports, gofiles...) if i == errPkg { if wantError && err == nil { t.err = fmt.Errorf("compilation succeeded unexpectedly\n%s", out) @@ -727,7 +738,7 @@ func (t *test) run() { return } for i, gofiles := range pkgs { - _, err := compileInDir(runcmd, longdir, flags, gofiles...) + _, err := compileInDir(runcmd, longdir, flags, localImports, gofiles...) // Allow this package compilation fail based on conditions below; // its errors were checked in previous case. if err != nil && !(wantError && action == "errorcheckandrundir" && i == len(pkgs)-2) { From bfdf74be12e2527d797968870564e8dafc2aacb5 Mon Sep 17 00:00:00 2001 From: Alberto Donizetti Date: Thu, 31 May 2018 12:04:07 +0000 Subject: [PATCH 051/187] Revert "testing: only compute b.N once when passed -count > 1" This reverts golang.org/cl/110775 Reason for revert: this is causing huge slow-dows on every run after the 1st, on various benchmarks, on multiple architectures (see Issue 25622 for details). It's just a nice-to-have little optimization, and we're near the 1st go1.11 beta release, so I'm reverting it. Fixes #25622 Change-Id: I758ade4af4abf764abd8336d404396992d11a0c6 Reviewed-on: https://go-review.googlesource.com/115535 Reviewed-by: Josh Bleecher Snyder --- src/cmd/go/go_test.go | 4 ++++ src/testing/benchmark.go | 47 +++++++++++++--------------------------- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index c32be94823086..0f86834079d15 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -4911,6 +4911,10 @@ func TestTestRegexps(t *testing.T) { x_test.go:15: LOG: Y running N=2000000000 --- BENCH: BenchmarkX/Y x_test.go:15: LOG: Y running N=1 + x_test.go:15: LOG: Y running N=100 + x_test.go:15: LOG: Y running N=10000 + x_test.go:15: LOG: Y running N=1000000 + x_test.go:15: LOG: Y running N=100000000 x_test.go:15: LOG: Y running N=2000000000 --- BENCH: BenchmarkX x_test.go:13: LOG: X running N=1 diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index bef1492cd6976..9c7b1be79e743 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -251,20 +251,27 @@ func (b *B) run() { b.context.processBench(b) // Must call doBench. } else { // Running func Benchmark. - b.doBench(0) + b.doBench() } } -func (b *B) doBench(hint int) BenchmarkResult { - go b.launch(hint) +func (b *B) doBench() BenchmarkResult { + go b.launch() <-b.signal return b.result } -// autodetectN runs the benchmark function, gradually increasing the -// number of iterations until the benchmark runs for the requested -// benchtime. -func (b *B) autodetectN() { +// launch launches the benchmark function. It gradually increases the number +// of benchmark iterations until the benchmark runs for the requested benchtime. +// launch is run by the doBench function as a separate goroutine. +// run1 must have been called on b. +func (b *B) launch() { + // Signal that we're done whether we return normally + // or by FailNow's runtime.Goexit. + defer func() { + b.signal <- true + }() + // Run the benchmark for at least the specified amount of time. d := b.benchTime for n := 1; !b.failed && b.duration < d && n < 1e9; { @@ -282,26 +289,6 @@ func (b *B) autodetectN() { n = roundUp(n) b.runN(n) } -} - -// launch launches the benchmark function for hintN iterations. If -// hintN == 0, it autodetects the number of benchmark iterations based -// on the requested benchtime. -// launch is run by the doBench function as a separate goroutine. -// run1 must have been called on b. -func (b *B) launch(hintN int) { - // Signal that we're done whether we return normally - // or by FailNow's runtime.Goexit. - defer func() { - b.signal <- true - }() - - if hintN == 0 { - b.autodetectN() - } else { - b.runN(hintN) - } - b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes} } @@ -439,7 +426,6 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e // processBench runs bench b for the configured CPU counts and prints the results. func (ctx *benchContext) processBench(b *B) { for i, procs := range cpuList { - var nHint int for j := uint(0); j < *count; j++ { runtime.GOMAXPROCS(procs) benchName := benchmarkName(b.name, procs) @@ -458,10 +444,7 @@ func (ctx *benchContext) processBench(b *B) { } b.run1() } - r := b.doBench(nHint) - if j == 0 { - nHint = b.N - } + r := b.doBench() if b.failed { // The output could be very long here, but probably isn't. // We print it all, regardless, because we don't want to trim the reason From 33cd4fba38b82693d7ffc62f50ea763694d4c8c3 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 22 May 2018 17:33:23 -0400 Subject: [PATCH 052/187] Revert "cmd/compile: ignore g register in liveness analysis" This reverts commit ea200340702cf3ccfac7c5db1f11bb65c80971c7 now that CL 114695 fixed the root cause of #25504. Change-Id: If437fc832983bd8793bde28ce0e2e64436a0596c Reviewed-on: https://go-review.googlesource.com/114087 Reviewed-by: David Chase --- src/cmd/compile/internal/gc/plive.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/cmd/compile/internal/gc/plive.go b/src/cmd/compile/internal/gc/plive.go index f3f1ca3d397e6..88b4380637092 100644 --- a/src/cmd/compile/internal/gc/plive.go +++ b/src/cmd/compile/internal/gc/plive.go @@ -461,17 +461,6 @@ func (lv *Liveness) regEffects(v *ssa.Value) (uevar, kill liveRegMask) { for _, reg := range regs[:nreg] { if reg.GCNum() == -1 { if ptrOnly { - if reg.String() == "g" { - // Issue #25504: Sometimes we - // spill and reload the g - // register, which this sees - // as a pointer load into the - // g register. The g register - // isn't a GP register and - // can't appear in register - // maps. Ignore it. - continue - } v.Fatalf("pointer in non-pointer register %v", reg) } else { continue From 08e2e880e72a173e6c1d3ff708fac2d3661ced89 Mon Sep 17 00:00:00 2001 From: Johan Brandhorst Date: Thu, 31 May 2018 10:12:32 +0000 Subject: [PATCH 053/187] net/http: use fake Transport network when running in Node Replaces the existing local loopback check with a check to see whether the program is being interpreted by Node. This means tests that are run with Node will use the fake network while still allowing users who are using js/wasm to talk to local networks. Updates #25506 Change-Id: I8bc3c6808fa29293b7ac5f77b186140c4ed90b51 GitHub-Last-Rev: 43d26af7bc716b7a01dd8f47d7a2c2a2df549489 GitHub-Pull-Request: golang/go#25663 Reviewed-on: https://go-review.googlesource.com/115495 Reviewed-by: Agniva De Sarker Reviewed-by: Brad Fitzpatrick --- src/net/http/roundtrip_js.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/net/http/roundtrip_js.go b/src/net/http/roundtrip_js.go index e60b7368df727..277fc7ed3bbb5 100644 --- a/src/net/http/roundtrip_js.go +++ b/src/net/http/roundtrip_js.go @@ -11,14 +11,15 @@ import ( "fmt" "io" "io/ioutil" - "net" + "os" + "path" "strconv" "syscall/js" ) // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (*Transport) RoundTrip(req *Request) (*Response, error) { - if useFakeNetwork(req) { + if useFakeNetwork() { return t.roundTrip(req) } headers := js.Global.Get("Headers").New() @@ -135,15 +136,8 @@ func (*Transport) RoundTrip(req *Request) (*Response, error) { // useFakeNetwork is used to determine whether the request is made // by a test and should be made to use the fake in-memory network. -func useFakeNetwork(req *Request) bool { - host, _, err := net.SplitHostPort(req.Host) - if err != nil { - host = req.Host - } - if ip := net.ParseIP(host); ip != nil { - return ip.IsLoopback(ip) - } - return host == "localhost" +func useFakeNetwork() bool { + return len(os.Args) > 0 && path.Base(os.Args[0]) == "node" } // streamReader implements an io.ReadCloser wrapper for ReadableStream. From c4f9fa1f8ee623558ed40d4d4cd3c616697cc77b Mon Sep 17 00:00:00 2001 From: Zhou Peng Date: Thu, 31 May 2018 06:52:19 +0000 Subject: [PATCH 054/187] reflect: reuse values during comparison of maps in DeepEqual Change-Id: I82f999b8ed9434321a361bf1bcbed7cf6ee4bee6 Reviewed-on: https://go-review.googlesource.com/115475 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/reflect/deepequal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reflect/deepequal.go b/src/reflect/deepequal.go index 2fdd6a3d82ba3..5b6694d3f0beb 100644 --- a/src/reflect/deepequal.go +++ b/src/reflect/deepequal.go @@ -116,7 +116,7 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { for _, k := range v1.MapKeys() { val1 := v1.MapIndex(k) val2 := v2.MapIndex(k) - if !val1.IsValid() || !val2.IsValid() || !deepValueEqual(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) { + if !val1.IsValid() || !val2.IsValid() || !deepValueEqual(val1, val2, visited, depth+1) { return false } } From 6b4828a206c2c5a5b6bf3cd4bd92b9530ecca5e5 Mon Sep 17 00:00:00 2001 From: Constantin Konstantinidis Date: Sat, 19 May 2018 13:59:29 +0200 Subject: [PATCH 055/187] encoding/asn1: fix returned type for an Object Identifier Unmarshal/Marshal/Unmarshal was not idempotent as the Object Identifier type was not returned through the interface. The limit case OID = 0 returns an error. The zero OID is 0.0 A test is fixed to use the Object Identifier type. Other related test are added. Fixes #11130 Change-Id: I15483a3126066c9b99cf5bd9c4b0cc15ec1d61ca Reviewed-on: https://go-review.googlesource.com/113837 Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot Reviewed-by: Filippo Valsorda --- src/encoding/asn1/asn1.go | 2 +- src/encoding/asn1/asn1_test.go | 2 +- src/encoding/asn1/marshal_test.go | 57 +++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/encoding/asn1/asn1.go b/src/encoding/asn1/asn1.go index ae382ee6bf52e..1ed357adfffcc 100644 --- a/src/encoding/asn1/asn1.go +++ b/src/encoding/asn1/asn1.go @@ -250,7 +250,7 @@ func (oi ObjectIdentifier) String() string { // parseObjectIdentifier parses an OBJECT IDENTIFIER from the given bytes and // returns it. An object identifier is a sequence of variable length integers // that are assigned in a hierarchy. -func parseObjectIdentifier(bytes []byte) (s []int, err error) { +func parseObjectIdentifier(bytes []byte) (s ObjectIdentifier, err error) { if len(bytes) == 0 { err = SyntaxError{"zero length OBJECT IDENTIFIER"} return diff --git a/src/encoding/asn1/asn1_test.go b/src/encoding/asn1/asn1_test.go index 185349773f593..f0a54e0cb2e69 100644 --- a/src/encoding/asn1/asn1_test.go +++ b/src/encoding/asn1/asn1_test.go @@ -227,7 +227,7 @@ func TestBitStringRightAlign(t *testing.T) { type objectIdentifierTest struct { in []byte ok bool - out []int + out ObjectIdentifier // has base type[]int } var objectIdentifierTestData = []objectIdentifierTest{ diff --git a/src/encoding/asn1/marshal_test.go b/src/encoding/asn1/marshal_test.go index f20ccdc8e98c5..b19b08b35243c 100644 --- a/src/encoding/asn1/marshal_test.go +++ b/src/encoding/asn1/marshal_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" "time" + "reflect" ) type intStruct struct { @@ -253,6 +254,62 @@ func TestInvalidUTF8(t *testing.T) { } } +func TestMarshalOID(t *testing.T) { + var marshalTestsOID = []marshalTest{ + {[]byte("\x06\x01\x30"), "0403060130"}, // bytes format returns a byte sequence \x04 + // {ObjectIdentifier([]int{0}), "060100"}, // returns an error as OID 0.0 has the same encoding + {[]byte("\x06\x010"), "0403060130"}, // same as above "\x06\x010" = "\x06\x01" + "0" + {ObjectIdentifier([]int{2,999,3}), "0603883703"}, // Example of ITU-T X.690 + {ObjectIdentifier([]int{0,0}), "060100"}, // zero OID + } + for i, test := range marshalTestsOID { + data, err := Marshal(test.in) + if err != nil { + t.Errorf("#%d failed: %s", i, err) + } + out, _ := hex.DecodeString(test.out) + if !bytes.Equal(out, data) { + t.Errorf("#%d got: %x want %x\n\t%q\n\t%q", i, data, out, data, out) + } + } +} + +func TestIssue11130(t *testing.T) { + data := []byte("\x06\x010") // == \x06\x01\x30 == OID = 0 (the figure) + var v interface{} + // v has Zero value here and Elem() would panic + _, err := Unmarshal(data, &v) + if err != nil { + t.Errorf("%v", err) + return + } + if reflect.TypeOf(v).String() != reflect.TypeOf(ObjectIdentifier{}).String() { + t.Errorf("marshal OID returned an invalid type") + return + } + + data1, err := Marshal(v) + if err != nil { + t.Errorf("%v", err) + return + } + + if !bytes.Equal(data,data1) { + t.Errorf("got: %q, want: %q \n", data1, data) + return + } + + var v1 interface{} + _, err = Unmarshal(data1, &v1) + if err != nil { + t.Errorf("%v", err) + return + } + if !reflect.DeepEqual(v, v1) { + t.Errorf("got: %#v data=%q , want : %#v data=%q\n ", v1, data1, v, data) + } +} + func BenchmarkMarshal(b *testing.B) { b.ReportAllocs() From 424c2157392d46220213610a290a275c36ddcd97 Mon Sep 17 00:00:00 2001 From: Hana Kim Date: Thu, 31 May 2018 13:16:24 -0400 Subject: [PATCH 056/187] cmd/pprof: fix help message formatting error Pprof usage message includes "%" symbols. Misuse of Fprintf caused the message to be interpreted as a format string and corrupted the usage message. Change-Id: I4732b491e2368cff9fdbfe070c125228d6f506fd Reviewed-on: https://go-review.googlesource.com/115595 Reviewed-by: Brad Fitzpatrick --- src/cmd/pprof/readlineui.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/pprof/readlineui.go b/src/cmd/pprof/readlineui.go index 67fb7aa49ca91..6e91816f9be1b 100644 --- a/src/cmd/pprof/readlineui.go +++ b/src/cmd/pprof/readlineui.go @@ -86,7 +86,7 @@ func (r *readlineUI) print(withColor bool, args ...interface{}) { if withColor { text = colorize(text) } - fmt.Fprintf(r.term, text) + fmt.Fprint(r.term, text) } // colorize prints the msg in red using ANSI color escapes. From 57d40f1b27c0e0a4ca491895a68efc40c7c7d435 Mon Sep 17 00:00:00 2001 From: Yury Smolsky Date: Thu, 31 May 2018 18:51:00 +0300 Subject: [PATCH 057/187] test: remove rundircmpout and cmpout actions This CL removes the rundircmpout action completely because it is not used anywhere. The run case already looks for output files. Rename the cmpout action mentioned in tests to the run action and remove "cmpout" from run.go. Change-Id: I835ceb70082927f8e9360e0ea0ba74f296363ab3 Reviewed-on: https://go-review.googlesource.com/115575 Run-TryBot: Yury Smolsky TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- misc/cgo/life/main.go | 2 +- misc/cgo/stdio/chain.go | 2 +- misc/cgo/stdio/fib.go | 2 +- misc/cgo/stdio/hello.go | 2 +- test/deferprint.go | 2 +- test/fixedbugs/bug206.go | 2 +- test/fixedbugs/bug328.go | 2 +- test/fixedbugs/bug409.go | 2 +- test/fixedbugs/issue21887.go | 2 +- test/fixedbugs/issue22683.go | 2 +- test/fixedbugs/issue25322.go | 2 +- test/fixedbugs/issue6899.go | 2 +- test/goprint.go | 2 +- test/helloworld.go | 2 +- test/ken/cplx0.go | 2 +- test/ken/string.go | 2 +- test/print.go | 2 +- test/printbig.go | 2 +- test/run.go | 4 ---- test/sigchld.go | 2 +- 20 files changed, 19 insertions(+), 23 deletions(-) diff --git a/misc/cgo/life/main.go b/misc/cgo/life/main.go index 45376fd05a97f..145a273bdd1fd 100644 --- a/misc/cgo/life/main.go +++ b/misc/cgo/life/main.go @@ -1,4 +1,4 @@ -// cmpout -tags=use_go_run +// run -tags=use_go_run // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/misc/cgo/stdio/chain.go b/misc/cgo/stdio/chain.go index 0fa813cab7082..cdc385208c989 100644 --- a/misc/cgo/stdio/chain.go +++ b/misc/cgo/stdio/chain.go @@ -1,4 +1,4 @@ -// cmpout -tags=use_go_run +// run -tags=use_go_run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/misc/cgo/stdio/fib.go b/misc/cgo/stdio/fib.go index 56e32552ee6de..58f185c90f051 100644 --- a/misc/cgo/stdio/fib.go +++ b/misc/cgo/stdio/fib.go @@ -1,4 +1,4 @@ -// cmpout -tags=use_go_run +// run -tags=use_go_run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/misc/cgo/stdio/hello.go b/misc/cgo/stdio/hello.go index 63bff4c617a13..56220d34be324 100644 --- a/misc/cgo/stdio/hello.go +++ b/misc/cgo/stdio/hello.go @@ -1,4 +1,4 @@ -// cmpout -tags=use_go_run +// run -tags=use_go_run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/deferprint.go b/test/deferprint.go index 3dc08542c1f76..b74677ac59c98 100644 --- a/test/deferprint.go +++ b/test/deferprint.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/bug206.go b/test/fixedbugs/bug206.go index c2382acf13ff5..91efa3ff79005 100644 --- a/test/fixedbugs/bug206.go +++ b/test/fixedbugs/bug206.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/bug328.go b/test/fixedbugs/bug328.go index 180af05fde5a9..57043f30af72d 100644 --- a/test/fixedbugs/bug328.go +++ b/test/fixedbugs/bug328.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/bug409.go b/test/fixedbugs/bug409.go index 9e08a8e67633c..e8546361ab493 100644 --- a/test/fixedbugs/bug409.go +++ b/test/fixedbugs/bug409.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/issue21887.go b/test/fixedbugs/issue21887.go index 9e3e91fcdb782..73c3f43596ff4 100644 --- a/test/fixedbugs/issue21887.go +++ b/test/fixedbugs/issue21887.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/issue22683.go b/test/fixedbugs/issue22683.go index a59a0edaf4f5c..47c7f6513d9f3 100644 --- a/test/fixedbugs/issue22683.go +++ b/test/fixedbugs/issue22683.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/issue25322.go b/test/fixedbugs/issue25322.go index 7489bbdfc2219..ee4ff53e2eedf 100644 --- a/test/fixedbugs/issue25322.go +++ b/test/fixedbugs/issue25322.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/issue6899.go b/test/fixedbugs/issue6899.go index f98f551b321a6..d7f8578029884 100644 --- a/test/fixedbugs/issue6899.go +++ b/test/fixedbugs/issue6899.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/goprint.go b/test/goprint.go index 0648c77e7d857..57eeac53a8036 100644 --- a/test/goprint.go +++ b/test/goprint.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/helloworld.go b/test/helloworld.go index 5025ec9bb350b..06851d13b3aa7 100644 --- a/test/helloworld.go +++ b/test/helloworld.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/ken/cplx0.go b/test/ken/cplx0.go index 665e52a5f3580..5d78dc0147a1e 100644 --- a/test/ken/cplx0.go +++ b/test/ken/cplx0.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/ken/string.go b/test/ken/string.go index 6df8dc4ddf193..7bb3cabbc2d12 100644 --- a/test/ken/string.go +++ b/test/ken/string.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/print.go b/test/print.go index b7f3db0a41ba7..7718c735e48d0 100644 --- a/test/print.go +++ b/test/print.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/printbig.go b/test/printbig.go index 5693c58d4f6a3..9e08c39adc232 100644 --- a/test/printbig.go +++ b/test/printbig.go @@ -1,4 +1,4 @@ -// cmpout +// run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/run.go b/test/run.go index 0805ecd4fcfe8..3cd19118778fb 100644 --- a/test/run.go +++ b/test/run.go @@ -502,10 +502,6 @@ func (t *test) run() { // TODO: Clean up/simplify this switch statement. switch action { - case "rundircmpout": - action = "rundir" - case "cmpout": - action = "run" // the run case already looks for /.out files case "compile", "compiledir", "build", "builddir", "buildrundir", "run", "buildrun", "runoutput", "rundir", "asmcheck": // nothing to do case "errorcheckandrundir": diff --git a/test/sigchld.go b/test/sigchld.go index 38437e5522b26..3b4960640914f 100644 --- a/test/sigchld.go +++ b/test/sigchld.go @@ -1,5 +1,5 @@ // +build !plan9,!windows -// cmpout +// run // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style From e57cdd81e25a8351a868679d0d7252928b6e5be4 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 24 May 2018 15:39:21 -0700 Subject: [PATCH 058/187] go/types: initial framework for marking-based cycle detection The existing code explicitly passes a (type name) path around to determine cycles; it also restarts the path for types that "break" a cycle (such as a pointer, function, etc.). This does not work for alias types (whose cycles are broken in a different way). Furthermore, because the path is not passed through all type checker functions that need it, we can't see the path or use it for detection of some cycles (e.g. cycles involving array lengths), which required ad-hoc solutions in those cases. This change introduces an explicit marking scheme for any kind of object; an object is painted in various colors indicating its state. It also introduces an object path (a stack) main- tained with the Checker state, which is available in all type checker functions that need access to it. The change only introduces these mechanisms and exercises the basic functionality, with no effect on the existing code for now. For #25141. Change-Id: I7c28714bdafe6c8d9afedf12a8a887554237337c Reviewed-on: https://go-review.googlesource.com/114517 Reviewed-by: Alan Donovan --- src/go/types/check.go | 16 +++++++++ src/go/types/decl.go | 78 ++++++++++++++++++++++++++++++++++++++-- src/go/types/object.go | 58 +++++++++++++++++++++++++----- src/go/types/universe.go | 3 +- 4 files changed, 143 insertions(+), 12 deletions(-) diff --git a/src/go/types/check.go b/src/go/types/check.go index 177065fded54e..1d75ab1fc73c4 100644 --- a/src/go/types/check.go +++ b/src/go/types/check.go @@ -90,6 +90,7 @@ type Checker struct { interfaces map[*TypeName]*ifaceInfo // maps interface type names to corresponding interface infos untyped map[ast.Expr]exprInfo // map of expressions without final type delayed []func() // stack of delayed actions + objPath []Object // path of object dependencies during type inference (for cycle reporting) // context within which the current object is type-checked // (valid only for the duration of type-checking a specific object) @@ -144,6 +145,21 @@ func (check *Checker) later(f func()) { check.delayed = append(check.delayed, f) } +// push pushes obj onto the object path and returns its index in the path. +func (check *Checker) push(obj Object) int { + check.objPath = append(check.objPath, obj) + return len(check.objPath) - 1 +} + +// pop pops and returns the topmost object from the object path. +func (check *Checker) pop() Object { + i := len(check.objPath) - 1 + obj := check.objPath[i] + check.objPath[i] = nil + check.objPath = check.objPath[:i] + return obj +} + // NewChecker returns a new Checker instance for a given package. // Package files may be added incrementally via checker.Files. func NewChecker(conf *Config, fset *token.FileSet, pkg *Package, info *Info) *Checker { diff --git a/src/go/types/decl.go b/src/go/types/decl.go index 288ba8e447916..aa769ce678cee 100644 --- a/src/go/types/decl.go +++ b/src/go/types/decl.go @@ -52,8 +52,82 @@ func pathString(path []*TypeName) string { // objDecl type-checks the declaration of obj in its respective (file) context. // See check.typ for the details on def and path. func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { - if obj.Type() != nil { - return // already checked - nothing to do + // Checking the declaration of obj means inferring its type + // (and possibly its value, for constants). + // An object's type (and thus the object) may be in one of + // three states which are expressed by colors: + // + // - an object whose type is not yet known is painted white (initial color) + // - an object whose type is in the process of being inferred is painted grey + // - an object whose type is fully inferred is painted black + // + // During type inference, an object's color changes from white to grey + // to black (pre-declared objects are painted black from the start). + // A black object (i.e., its type) can only depend on (refer to) other black + // ones. White and grey objects may depend on white and black objects. + // A dependency on a grey object indicates a cycle which may or may not be + // valid. + // + // When objects turn grey, they are pushed on the object path (a stack); + // they are popped again when they turn black. Thus, if a grey object (a + // cycle) is encountered, it is on the object path, and all the objects + // it depends on are the remaining objects on that path. Color encoding + // is such that the color value of a grey object indicates the index of + // that object in the object path. + + // During type-checking, white objects may be assigned a type without + // traversing through objDecl; e.g., when initializing constants and + // variables. Update the colors of those objects here (rather than + // everywhere where we set the type) to satisfy the color invariants. + if obj.color() == white && obj.Type() != nil { + obj.setColor(black) + return + } + + switch obj.color() { + case white: + assert(obj.Type() == nil) + // All color values other than white and black are considered grey. + // Because black and white are < grey, all values >= grey are grey. + // Use those values to encode the object's index into the object path. + obj.setColor(grey + color(check.push(obj))) + defer func() { + check.pop().setColor(black) + }() + + case black: + assert(obj.Type() != nil) + return + + default: + // Color values other than white or black are considered grey. + fallthrough + + case grey: + // We have a cycle. + // In the existing code, this is marked by a non-nil type + // for the object except for constants and variables, which + // have their own "visited" flag (the new marking approach + // will allow us to remove that flag eventually). Their type + // may be nil because they haven't determined their init + // values yet (from which to deduce the type). But in that + // case, they must have been marked as visited. + // For now, handle constants and variables specially. + visited := false + switch obj := obj.(type) { + case *Const: + visited = obj.visited + case *Var: + visited = obj.visited + default: + assert(obj.Type() != nil) + return + } + if obj.Type() != nil { + return + } + assert(visited) + } if trace { diff --git a/src/go/types/object.go b/src/go/types/object.go index f158e2733f337..1305a9db6eb98 100644 --- a/src/go/types/object.go +++ b/src/go/types/object.go @@ -34,9 +34,15 @@ type Object interface { // 0 for all other objects (including objects in file scopes). order() uint32 + // color returns the object's color. + color() color + // setOrder sets the order number of the object. It must be > 0. setOrder(uint32) + // setColor sets the object's color. It must not be white. + setColor(color color) + // setParent sets the parent scope of the object. setParent(*Scope) @@ -78,9 +84,41 @@ type object struct { name string typ Type order_ uint32 + color_ color scopePos_ token.Pos } +// color encodes the color of an object (see Checker.objDecl for details). +type color uint32 + +// An object may be painted in one of three colors. +// Color values other than white or black are considered grey. +const ( + white color = iota + black + grey // must be > white and black +) + +func (c color) String() string { + switch c { + case white: + return "white" + case black: + return "black" + default: + return "grey" + } +} + +// colorFor returns the (initial) color for an object depending on +// whether its type t is known or not. +func colorFor(t Type) color { + if t != nil { + return black + } + return white +} + // Parent returns the scope in which the object is declared. // The result is nil for methods and struct fields. func (obj *object) Parent() *Scope { return obj.parent } @@ -108,10 +146,12 @@ func (obj *object) Id() string { return Id(obj.pkg, obj.name) } func (obj *object) String() string { panic("abstract") } func (obj *object) order() uint32 { return obj.order_ } +func (obj *object) color() color { return obj.color_ } func (obj *object) scopePos() token.Pos { return obj.scopePos_ } func (obj *object) setParent(parent *Scope) { obj.parent = parent } func (obj *object) setOrder(order uint32) { assert(order > 0); obj.order_ = order } +func (obj *object) setColor(color color) { assert(color != white); obj.color_ = color } func (obj *object) setScopePos(pos token.Pos) { obj.scopePos_ = pos } func (obj *object) sameId(pkg *Package, name string) bool { @@ -147,7 +187,7 @@ type PkgName struct { // NewPkgName returns a new PkgName object representing an imported package. // The remaining arguments set the attributes found with all Objects. func NewPkgName(pos token.Pos, pkg *Package, name string, imported *Package) *PkgName { - return &PkgName{object{nil, pos, pkg, name, Typ[Invalid], 0, token.NoPos}, imported, false} + return &PkgName{object{nil, pos, pkg, name, Typ[Invalid], 0, black, token.NoPos}, imported, false} } // Imported returns the package that was imported. @@ -164,7 +204,7 @@ type Const struct { // NewConst returns a new constant with value val. // The remaining arguments set the attributes found with all Objects. func NewConst(pos token.Pos, pkg *Package, name string, typ Type, val constant.Value) *Const { - return &Const{object{nil, pos, pkg, name, typ, 0, token.NoPos}, val, false} + return &Const{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, val, false} } // Val returns the constant's value. @@ -185,7 +225,7 @@ type TypeName struct { // argument for NewNamed, which will set the TypeName's type as a side- // effect. func NewTypeName(pos token.Pos, pkg *Package, name string, typ Type) *TypeName { - return &TypeName{object{nil, pos, pkg, name, typ, 0, token.NoPos}} + return &TypeName{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}} } // IsAlias reports whether obj is an alias name for a type. @@ -224,19 +264,19 @@ type Var struct { // NewVar returns a new variable. // The arguments set the attributes found with all Objects. func NewVar(pos token.Pos, pkg *Package, name string, typ Type) *Var { - return &Var{object: object{nil, pos, pkg, name, typ, 0, token.NoPos}} + return &Var{object: object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}} } // NewParam returns a new variable representing a function parameter. func NewParam(pos token.Pos, pkg *Package, name string, typ Type) *Var { - return &Var{object: object{nil, pos, pkg, name, typ, 0, token.NoPos}, used: true} // parameters are always 'used' + return &Var{object: object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, used: true} // parameters are always 'used' } // NewField returns a new variable representing a struct field. // For embedded fields, the name is the unqualified type name /// under which the field is accessible. func NewField(pos token.Pos, pkg *Package, name string, typ Type, embedded bool) *Var { - return &Var{object: object{nil, pos, pkg, name, typ, 0, token.NoPos}, embedded: embedded, isField: true} + return &Var{object: object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, embedded: embedded, isField: true} } // Anonymous reports whether the variable is an embedded field. @@ -266,7 +306,7 @@ func NewFunc(pos token.Pos, pkg *Package, name string, sig *Signature) *Func { if sig != nil { typ = sig } - return &Func{object{nil, pos, pkg, name, typ, 0, token.NoPos}} + return &Func{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}} } // FullName returns the package- or receiver-type-qualified name of @@ -291,7 +331,7 @@ type Label struct { // NewLabel returns a new label. func NewLabel(pos token.Pos, pkg *Package, name string) *Label { - return &Label{object{pos: pos, pkg: pkg, name: name, typ: Typ[Invalid]}, false} + return &Label{object{pos: pos, pkg: pkg, name: name, typ: Typ[Invalid], color_: black}, false} } // A Builtin represents a built-in function. @@ -302,7 +342,7 @@ type Builtin struct { } func newBuiltin(id builtinId) *Builtin { - return &Builtin{object{name: predeclaredFuncs[id].name, typ: Typ[Invalid]}, id} + return &Builtin{object{name: predeclaredFuncs[id].name, typ: Typ[Invalid], color_: black}, id} } // Nil represents the predeclared value nil. diff --git a/src/go/types/universe.go b/src/go/types/universe.go index 286ef7ba46854..2ae8a319707c4 100644 --- a/src/go/types/universe.go +++ b/src/go/types/universe.go @@ -102,7 +102,7 @@ func defPredeclaredConsts() { } func defPredeclaredNil() { - def(&Nil{object{name: "nil", typ: Typ[UntypedNil]}}) + def(&Nil{object{name: "nil", typ: Typ[UntypedNil], color_: black}}) } // A builtinId is the id of a builtin function. @@ -207,6 +207,7 @@ func init() { // scope; other objects are inserted in the universe scope. // func def(obj Object) { + assert(obj.color() == black) name := obj.Name() if strings.Contains(name, " ") { return // nothing to do From 462c182ce7d6ee16af9731e7d14da2cb9be6a91a Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Wed, 30 May 2018 13:48:13 -0700 Subject: [PATCH 059/187] go/types: use color-marking based cycle detection at package level The existing cycle detection scheme passes around a (type name) path; when a type name re-appears in the path, a cycle is reported. Indirections (as in *T, func(T), etc.) are broken by starting a new (nil) path. The problem with this approach is that it doesn't work for cycles involving alias type names since they may be invalid even if there is an indirection. Furthermore, the path must be passed around through all functions which is currently not the case, which leads to less optimial error reporting. The new code is using the previously introduced color marking scheme and global object path for package-level cycle detection (function-local cycle detection doesn't use the same code path yet but is also much less important as cycles can only be created using the type being declared). The new code is guarded with an internal flag (useCycleMarking) so it can be disabled in short notice if this change introduced unexpected new issues. Fixes #23139. Fixes #25141. For #18640. For #24939. Change-Id: I1bbf2d2d61a375cf5885b2de1df0a9819d63e5fa Reviewed-on: https://go-review.googlesource.com/115455 Reviewed-by: Alan Donovan --- src/go/types/decl.go | 85 ++++++++++++++++++++++++++++++- src/go/types/testdata/cycles5.src | 39 +++++++++++++- src/go/types/typexpr.go | 18 ++++++- 3 files changed, 137 insertions(+), 5 deletions(-) diff --git a/src/go/types/decl.go b/src/go/types/decl.go index aa769ce678cee..b1543e8a11215 100644 --- a/src/go/types/decl.go +++ b/src/go/types/decl.go @@ -49,6 +49,13 @@ func pathString(path []*TypeName) string { return s } +// useCycleMarking enables the new coloring-based cycle marking scheme +// for package-level objects. Set this flag to false to disable this +// code quickly and revert to the existing mechanism (and comment out +// some of the new tests in cycles5.src that will fail again). +// TODO(gri) remove this for Go 1.12 +const useCycleMarking = true + // objDecl type-checks the declaration of obj in its respective (file) context. // See check.typ for the details on def and path. func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { @@ -117,12 +124,32 @@ func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { switch obj := obj.(type) { case *Const: visited = obj.visited + case *Var: visited = obj.visited - default: + + case *TypeName: + assert(obj.Type() != nil) + if useCycleMarking { + check.typeCycle(obj) + } + return + + case *Func: + // Cycles involving functions require variables in + // the cycle; they are pretty esoteric. For now we + // handle this as before (for grey functions, the + // function type is set to an empty signature which + // makes it impossible to initialize a variable with + // the function). assert(obj.Type() != nil) return + + default: + unreachable() } + + // we have a *Const or *Var if obj.Type() != nil { return } @@ -176,6 +203,60 @@ func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { } } +// indir is a sentinel type name that is pushed onto the object path +// to indicate an "indirection" in the dependency from one type name +// to the next. For instance, for "type p *p" the object path contains +// p followed by indir, indicating that there's an indirection *p. +// Indirections are used to break type cycles. +var indir = new(TypeName) + +// typeCycle checks if the cycle starting with obj is valid and +// reports an error if it is not. +func (check *Checker) typeCycle(obj *TypeName) { + d := check.objMap[obj] + if d == nil { + check.dump("%v: %s should have been declared", obj.Pos(), obj) + unreachable() + } + + // A cycle must have at least one indirection and one defined + // type to be permitted: If there is no indirection, the size + // of the type cannot be computed (it's either infinite or 0); + // if there is no defined type, we have a sequence of alias + // type names which will expand ad infinitum. + var hasIndir, hasDefType bool + assert(obj.color() >= grey) + start := obj.color() - grey // index of obj in objPath + cycle := check.objPath[start:] + for _, obj := range cycle { + // Cycles may contain various objects; for now only look at type names. + if tname, _ := obj.(*TypeName); tname != nil { + if tname == indir { + hasIndir = true + } else if !check.objMap[tname].alias { + hasDefType = true + } + if hasIndir && hasDefType { + return // cycle is permitted + } + } + } + + // break cycle + // (without this, calling underlying() below may lead to an endless loop) + obj.typ = Typ[Invalid] + + // report cycle + check.errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name()) + for _, obj := range cycle { + if obj == indir { + continue // don't print indir sentinels + } + check.errorf(obj.Pos(), "\t%s refers to", obj.Name()) // secondary error, \t indented + } + check.errorf(obj.Pos(), "\t%s", obj.Name()) +} + func (check *Checker) constDecl(obj *Const, typ, init ast.Expr) { assert(obj.typ == nil) @@ -353,7 +434,7 @@ func (check *Checker) addMethodDecls(obj *TypeName) { return } delete(check.methods, obj) - assert(!obj.IsAlias()) + assert(!check.objMap[obj].alias) // don't use TypeName.IsAlias (requires fully set up object) // use an objset to check for name conflicts var mset objset diff --git a/src/go/types/testdata/cycles5.src b/src/go/types/testdata/cycles5.src index aab9ee235ebfb..3fa62af5b1fca 100644 --- a/src/go/types/testdata/cycles5.src +++ b/src/go/types/testdata/cycles5.src @@ -97,12 +97,12 @@ var _ = err.Error() // more esoteric cases type ( - T1 interface { T2 /* ERROR not an interface */ } + T1 interface { T2 } T2 /* ERROR cycle */ T2 ) type ( - T3 interface { T4 /* ERROR not an interface */ } + T3 interface { T4 } T4 /* ERROR cycle */ T5 T5 = T6 T6 = T7 @@ -117,3 +117,38 @@ const n = unsafe.Sizeof(func(){}) type I interface { m([unsafe.Sizeof(func() { I.m(nil, [n]byte{}) })]byte) } + + +// test cases for varias alias cycles + +type T10 /* ERROR cycle */ = *T10 // issue #25141 +type T11 /* ERROR cycle */ = interface{ f(T11) } // issue #23139 + +// issue #18640 +type ( + aa = bb + bb struct { + *aa + } +) + +type ( + a struct{ *b } + b = c + c struct{ *b } +) + +// issue #24939 +type ( + _ interface { + M(P) + } + + M interface { + F() P + } + + P = interface { + I() M + } +) diff --git a/src/go/types/typexpr.go b/src/go/types/typexpr.go index d3841c93676db..e3f50000ec807 100644 --- a/src/go/types/typexpr.go +++ b/src/go/types/typexpr.go @@ -71,6 +71,12 @@ func (check *Checker) ident(x *operand, e *ast.Ident, def *Named, path []*TypeNa case *TypeName: x.mode = typexpr + // package-level alias cycles are now checked by Checker.objDecl + if useCycleMarking { + if check.objMap[obj] != nil { + break + } + } if check.cycle(obj, path, true) { // maintain x.mode == typexpr despite error typ = Typ[Invalid] @@ -132,7 +138,11 @@ func (check *Checker) cycle(obj *TypeName, path []*TypeName, report bool) bool { // If def != nil, e is the type specification for the named type def, declared // in a type declaration, and def.underlying will be set to the type of e before // any components of e are type-checked. Path contains the path of named types -// referring to this type. +// referring to this type; i.e. it is the path of named types directly containing +// each other and leading to the current type e. Indirect containment (e.g. via +// pointer indirection, function parameter, etc.) breaks the path (leads to a new +// path, and usually via calling Checker.typ below) and those types are not found +// in the path. // func (check *Checker) typExpr(e ast.Expr, def *Named, path []*TypeName) (T Type) { if trace { @@ -152,6 +162,12 @@ func (check *Checker) typExpr(e ast.Expr, def *Named, path []*TypeName) (T Type) } func (check *Checker) typ(e ast.Expr) Type { + // typExpr is called with a nil path indicating an indirection: + // push indir sentinel on object path + if useCycleMarking { + check.push(indir) + defer check.pop() + } return check.typExpr(e, nil, nil) } From 6dbaf0352a911f2f4c835dcfbf32aeb38f0b4462 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Tue, 8 May 2018 18:01:16 -0700 Subject: [PATCH 060/187] go/types: better cycle reporting for some cyclic composite literals To evaluate the type of composite literals, the type checker called Checker.typ which breaks cycles. As a result, certain cycles were not reported with actual cycle reporting, but caught due to other uninitialized fields (with less nice error message). The change now calls Checker.typExpr at the relevant call site. For #18643. Change-Id: Iecb3f0e1afb4585b85553b6c581212f52ac3a1c4 Reviewed-on: https://go-review.googlesource.com/115456 Reviewed-by: Alan Donovan --- src/go/types/builtins.go | 2 +- src/go/types/expr.go | 2 +- src/go/types/testdata/cycles.src | 2 +- src/go/types/typexpr.go | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/go/types/builtins.go b/src/go/types/builtins.go index afe5f5d0fcb44..05e032423ca2c 100644 --- a/src/go/types/builtins.go +++ b/src/go/types/builtins.go @@ -174,7 +174,7 @@ func (check *Checker) builtin(x *operand, call *ast.CallExpr, id builtinId) (_ b } } - if mode == invalid { + if mode == invalid && typ != Typ[Invalid] { check.invalidArg(x.pos(), "%s for %s", x, bin.name) return } diff --git a/src/go/types/expr.go b/src/go/types/expr.go index 0a2a811bd8898..3f3c4f83c6489 100644 --- a/src/go/types/expr.go +++ b/src/go/types/expr.go @@ -1064,7 +1064,7 @@ func (check *Checker) exprInternal(x *operand, e ast.Expr, hint Type) exprKind { break } } - typ = check.typ(e.Type) + typ = check.typExpr(e.Type, nil, nil) base = typ case hint != nil: diff --git a/src/go/types/testdata/cycles.src b/src/go/types/testdata/cycles.src index 79e75e9316d7f..59f112dba1db7 100644 --- a/src/go/types/testdata/cycles.src +++ b/src/go/types/testdata/cycles.src @@ -147,7 +147,7 @@ type ( // test cases for issue 18643 // (type cycle detection when non-type expressions are involved) type ( - T14 [len(T14 /* ERROR cycle */ {})]int + T14 /* ERROR cycle */ [len(T14{})]int T15 [][len(T15 /* ERROR cycle */ {})]int T16 map[[len(T16 /* ERROR cycle */ {1:2})]int]int T17 map[int][len(T17 /* ERROR cycle */ {1:2})]int diff --git a/src/go/types/typexpr.go b/src/go/types/typexpr.go index e3f50000ec807..45ada5874bc19 100644 --- a/src/go/types/typexpr.go +++ b/src/go/types/typexpr.go @@ -161,6 +161,11 @@ func (check *Checker) typExpr(e ast.Expr, def *Named, path []*TypeName) (T Type) return } +// typ is like typExpr (with a nil argument for the def parameter), +// but typ breaks type cycles. It should be called for components of +// types that break cycles, such as pointer base types, slice or map +// element types, etc. See the comment in typExpr for details. +// func (check *Checker) typ(e ast.Expr) Type { // typExpr is called with a nil path indicating an indirection: // push indir sentinel on object path From e4259d67b9e1f0180a923faa512a1781465faac4 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Wed, 30 May 2018 18:13:27 -0700 Subject: [PATCH 061/187] go/types: report object path in trace mode For debugging only; disabled (dead code) by default unless internal constant trace flag is set to true. For #8699. Change-Id: Ib7b272c6ac8efacccbbbe24650ef500c5a9ddcf5 Reviewed-on: https://go-review.googlesource.com/115457 Reviewed-by: Alan Donovan --- src/go/types/check.go | 12 ++++++++++++ src/go/types/decl.go | 4 ++-- src/go/types/interfaces.go | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/go/types/check.go b/src/go/types/check.go index 1d75ab1fc73c4..286b1f36a9578 100644 --- a/src/go/types/check.go +++ b/src/go/types/check.go @@ -160,6 +160,18 @@ func (check *Checker) pop() Object { return obj } +// pathString returns a string of the form a->b-> ... ->g for an object path [a, b, ... g]. +func (check *Checker) pathString() string { + var s string + for i, p := range check.objPath { + if i > 0 { + s += "->" + } + s += p.Name() + } + return s +} + // NewChecker returns a new Checker instance for a given package. // Package files may be added incrementally via checker.Files. func NewChecker(conf *Config, fset *token.FileSet, pkg *Package, info *Info) *Checker { diff --git a/src/go/types/decl.go b/src/go/types/decl.go index b1543e8a11215..9a27fbbed6f0c 100644 --- a/src/go/types/decl.go +++ b/src/go/types/decl.go @@ -158,7 +158,7 @@ func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { } if trace { - check.trace(obj.Pos(), "-- checking %s (path = %s)", obj, pathString(path)) + check.trace(obj.Pos(), "-- checking %s (path = %s, objPath = %s)", obj, pathString(path), check.pathString()) check.indent++ defer func() { check.indent-- @@ -208,7 +208,7 @@ func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { // to the next. For instance, for "type p *p" the object path contains // p followed by indir, indicating that there's an indirection *p. // Indirections are used to break type cycles. -var indir = new(TypeName) +var indir = NewTypeName(token.NoPos, nil, "*", nil) // typeCycle checks if the cycle starting with obj is valid and // reports an error if it is not. diff --git a/src/go/types/interfaces.go b/src/go/types/interfaces.go index b4efebae5d059..e4b42dc5a36b9 100644 --- a/src/go/types/interfaces.go +++ b/src/go/types/interfaces.go @@ -144,7 +144,7 @@ func (check *Checker) infoFromTypeLit(scope *Scope, iface *ast.InterfaceType, tn } if trace { - check.trace(iface.Pos(), "-- collect methods for %v (path = %s)", iface, pathString(path)) + check.trace(iface.Pos(), "-- collect methods for %v (path = %s, objPath = %s)", iface, pathString(path), check.pathString()) check.indent++ defer func() { check.indent-- From a34e6650c0847ba54445b036dfd33e7d98fe8a2c Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Thu, 31 May 2018 12:30:19 -0700 Subject: [PATCH 062/187] encoding/hex: improve Decode and DecodeString docs Simplify the wording of both. Make the DecodeString docs more accurate: DecodeString returns a slice, not a string. Change-Id: Iba7003f55fb0a37aafcbeee59a30492c0f68aa4e Reviewed-on: https://go-review.googlesource.com/115615 Reviewed-by: Ian Lance Taylor --- src/encoding/hex/hex.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/encoding/hex/hex.go b/src/encoding/hex/hex.go index 4cb26b6673442..aee5aecb1a757 100644 --- a/src/encoding/hex/hex.go +++ b/src/encoding/hex/hex.go @@ -50,8 +50,8 @@ func DecodedLen(x int) int { return x / 2 } // Decode decodes src into DecodedLen(len(src)) bytes, // returning the actual number of bytes written to dst. // -// Decode expects that src contain only hexadecimal -// characters and that src should have an even length. +// Decode expects that src contains only hexadecimal +// characters and that src has even length. // If the input is malformed, Decode returns the number // of bytes decoded before the error. func Decode(dst, src []byte) (int, error) { @@ -101,10 +101,10 @@ func EncodeToString(src []byte) string { // DecodeString returns the bytes represented by the hexadecimal string s. // -// DecodeString expects that src contain only hexadecimal -// characters and that src should have an even length. -// If the input is malformed, DecodeString returns a string -// containing the bytes decoded before the error. +// DecodeString expects that src contains only hexadecimal +// characters and that src has even length. +// If the input is malformed, DecodeString returns +// the bytes decoded before the error. func DecodeString(s string) ([]byte, error) { src := []byte(s) // We can use the source slice itself as the destination From a7e0a920ad45482183783c56e4dd39c9457ff4cc Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 31 May 2018 15:07:02 -0700 Subject: [PATCH 063/187] archive/zip: remove unnecessary words in (*Writer).Close docs Fixes #25599 Change-Id: I19ac3463682f662515feaf4c6132f55c12ba5386 Reviewed-on: https://go-review.googlesource.com/115618 Reviewed-by: Brad Fitzpatrick --- src/archive/zip/writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archive/zip/writer.go b/src/archive/zip/writer.go index 0f1a193345b46..506148ee30089 100644 --- a/src/archive/zip/writer.go +++ b/src/archive/zip/writer.go @@ -72,7 +72,7 @@ func (w *Writer) SetComment(comment string) error { } // Close finishes writing the zip file by writing the central directory. -// It does not (and cannot) close the underlying writer. +// It does not close the underlying writer. func (w *Writer) Close() error { if w.last != nil && !w.last.closed { if err := w.last.close(); err != nil { From 063f97a6110079c2aaeb5f2c2a51d7f1bc7445ab Mon Sep 17 00:00:00 2001 From: Richard Musiol Date: Sun, 4 Mar 2018 12:16:18 +0100 Subject: [PATCH 064/187] os: add js/wasm architecture This commit adds the js/wasm architecture to the os package. Access to the actual file system is supported through Node.js. Updates #18892 Change-Id: I6fa642fb294ca020b2c545649d4324d981aa0408 Reviewed-on: https://go-review.googlesource.com/109977 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- misc/wasm/wasm_exec.js | 5 + src/internal/poll/fd_poll_nacljs.go | 5 +- src/internal/poll/fd_posix.go | 2 +- src/internal/poll/fd_unix.go | 2 +- src/internal/poll/sys_cloexec.go | 2 +- src/internal/syscall/unix/nonblocking_js.go | 9 + src/os/dir_unix.go | 2 +- src/os/error_posix.go | 2 +- src/os/error_unix.go | 2 +- src/os/error_unix_test.go | 2 +- src/os/exec/lp_js.go | 23 + src/os/exec_posix.go | 2 +- src/os/exec_unix.go | 2 +- src/os/executable_procfs.go | 2 +- src/os/file_posix.go | 2 +- src/os/file_unix.go | 2 +- src/os/os_test.go | 4 +- src/os/path_unix.go | 2 +- src/os/pipe_bsd.go | 2 +- src/os/pipe_test.go | 2 +- src/os/signal/signal_unix.go | 2 +- src/os/{stat_nacl.go => stat_nacljs.go} | 2 + src/os/stat_unix.go | 2 +- src/os/sys_bsd.go | 2 +- src/os/sys_js.go | 11 + src/os/timeout_test.go | 1 + src/os/user/lookup_unix.go | 2 +- src/os/wait_unimp.go | 2 +- src/path/filepath/path_unix.go | 2 +- src/runtime/debug/heapdump_test.go | 8 +- src/runtime/os_js.go | 145 ++++++ src/runtime/sys_wasm.s | 4 + src/runtime/trace/trace_test.go | 6 + src/syscall/dirent.go | 2 +- src/syscall/endian_little.go | 2 +- src/syscall/env_unix.go | 2 +- src/syscall/fs_js.go | 497 ++++++++++++++++++++ src/syscall/syscall_js.go | 308 ++++++++++++ src/syscall/tables_nacljs.go | 131 +++++- 39 files changed, 1175 insertions(+), 32 deletions(-) create mode 100644 src/internal/syscall/unix/nonblocking_js.go create mode 100644 src/os/exec/lp_js.go rename src/os/{stat_nacl.go => stat_nacljs.go} (98%) create mode 100644 src/os/sys_js.go create mode 100644 src/runtime/os_js.go create mode 100644 src/syscall/fs_js.go create mode 100644 src/syscall/syscall_js.go diff --git a/misc/wasm/wasm_exec.js b/misc/wasm/wasm_exec.js index 142080bf55169..e579ecf677d95 100755 --- a/misc/wasm/wasm_exec.js +++ b/misc/wasm/wasm_exec.js @@ -143,6 +143,11 @@ mem().setInt32(sp + 16, (msec % 1000) * 1000000, true); }, + // func getRandomData(r []byte) + "runtime.getRandomData": (sp) => { + crypto.getRandomValues(loadSlice(sp + 8)); + }, + // func boolVal(value bool) Value "syscall/js.boolVal": (sp) => { storeValue(sp + 16, mem().getUint8(sp + 8) !== 0); diff --git a/src/internal/poll/fd_poll_nacljs.go b/src/internal/poll/fd_poll_nacljs.go index 2701199ce7718..832dddb4aa447 100644 --- a/src/internal/poll/fd_poll_nacljs.go +++ b/src/internal/poll/fd_poll_nacljs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl +// +build nacl js,wasm package poll @@ -42,6 +42,9 @@ func (pd *pollDesc) wait(mode int, isFile bool) error { if pd.closing { return errClosing(isFile) } + if isFile { // TODO(neelance): wasm: Use callbacks from JS to block until the read/write finished. + return nil + } return ErrTimeout } diff --git a/src/internal/poll/fd_posix.go b/src/internal/poll/fd_posix.go index e0e634cdb2900..f178a6fa0ae1b 100644 --- a/src/internal/poll/fd_posix.go +++ b/src/internal/poll/fd_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows package poll diff --git a/src/internal/poll/fd_unix.go b/src/internal/poll/fd_unix.go index 36376ef6cbb78..5a196e7efec57 100644 --- a/src/internal/poll/fd_unix.go +++ b/src/internal/poll/fd_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package poll diff --git a/src/internal/poll/sys_cloexec.go b/src/internal/poll/sys_cloexec.go index 3a25b13bb5a22..7bafa0d81a1c4 100644 --- a/src/internal/poll/sys_cloexec.go +++ b/src/internal/poll/sys_cloexec.go @@ -5,7 +5,7 @@ // This file implements sysSocket and accept for platforms that do not // provide a fast path for setting SetNonblock and CloseOnExec. -// +build darwin nacl solaris +// +build darwin js,wasm nacl solaris package poll diff --git a/src/internal/syscall/unix/nonblocking_js.go b/src/internal/syscall/unix/nonblocking_js.go new file mode 100644 index 0000000000000..ff67c75e81a02 --- /dev/null +++ b/src/internal/syscall/unix/nonblocking_js.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +func IsNonblock(fd int) (nonblocking bool, err error) { + return false, nil +} diff --git a/src/os/dir_unix.go b/src/os/dir_unix.go index 3424688e8cfea..09c3d2eb61cc5 100644 --- a/src/os/dir_unix.go +++ b/src/os/dir_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package os diff --git a/src/os/error_posix.go b/src/os/error_posix.go index 2049e448e8fe8..3c81b41706fad 100644 --- a/src/os/error_posix.go +++ b/src/os/error_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows package os diff --git a/src/os/error_unix.go b/src/os/error_unix.go index be1440cacba83..a9d798b391533 100644 --- a/src/os/error_unix.go +++ b/src/os/error_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package os diff --git a/src/os/error_unix_test.go b/src/os/error_unix_test.go index 76fe015b22120..8db98676d106b 100644 --- a/src/os/error_unix_test.go +++ b/src/os/error_unix_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package os_test diff --git a/src/os/exec/lp_js.go b/src/os/exec/lp_js.go new file mode 100644 index 0000000000000..6750fb99b0f6a --- /dev/null +++ b/src/os/exec/lp_js.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package exec + +import ( + "errors" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath(file string) (string, error) { + // Wasm can not execute processes, so act as if there are no executables at all. + return "", &Error{file, ErrNotFound} +} diff --git a/src/os/exec_posix.go b/src/os/exec_posix.go index fb220c8a5a27d..ec5cf33236068 100644 --- a/src/os/exec_posix.go +++ b/src/os/exec_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows package os diff --git a/src/os/exec_unix.go b/src/os/exec_unix.go index c4999db57f090..b07543e550a08 100644 --- a/src/os/exec_unix.go +++ b/src/os/exec_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package os diff --git a/src/os/executable_procfs.go b/src/os/executable_procfs.go index b5fae59046862..5bb63b9bdc5c1 100644 --- a/src/os/executable_procfs.go +++ b/src/os/executable_procfs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux netbsd dragonfly nacl +// +build linux netbsd dragonfly nacl js,wasm package os diff --git a/src/os/file_posix.go b/src/os/file_posix.go index b8835a70b8e33..7cfafc8fde395 100644 --- a/src/os/file_posix.go +++ b/src/os/file_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows package os diff --git a/src/os/file_unix.go b/src/os/file_unix.go index ed7e8cb31c1d0..164d0c59779e0 100644 --- a/src/os/file_unix.go +++ b/src/os/file_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package os diff --git a/src/os/os_test.go b/src/os/os_test.go index 9d13fe05accc2..894105a886011 100644 --- a/src/os/os_test.go +++ b/src/os/os_test.go @@ -1390,7 +1390,7 @@ func TestSeek(t *testing.T) { func TestSeekError(t *testing.T) { switch runtime.GOOS { - case "plan9", "nacl": + case "js", "nacl", "plan9": t.Skipf("skipping test on %v", runtime.GOOS) } @@ -2252,6 +2252,8 @@ func TestPipeThreads(t *testing.T) { t.Skip("skipping on Windows; issue 19098") case "plan9": t.Skip("skipping on Plan 9; does not support runtime poller") + case "js": + t.Skip("skipping on js; no support for os.Pipe") } threads := 100 diff --git a/src/os/path_unix.go b/src/os/path_unix.go index 9117ad0ef6681..b2e0bca0df1eb 100644 --- a/src/os/path_unix.go +++ b/src/os/path_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package os diff --git a/src/os/pipe_bsd.go b/src/os/pipe_bsd.go index 5260ceea94f95..9735988f324d3 100644 --- a/src/os/pipe_bsd.go +++ b/src/os/pipe_bsd.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly nacl solaris +// +build darwin dragonfly js,wasm nacl solaris package os diff --git a/src/os/pipe_test.go b/src/os/pipe_test.go index aad6c27f1b11e..929e9bec5322e 100644 --- a/src/os/pipe_test.go +++ b/src/os/pipe_test.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // Test broken pipes on Unix systems. -// +build !windows,!plan9,!nacl +// +build !windows,!plan9,!nacl,!js package os_test diff --git a/src/os/signal/signal_unix.go b/src/os/signal/signal_unix.go index 0987c1730a70e..28fbb5499597d 100644 --- a/src/os/signal/signal_unix.go +++ b/src/os/signal/signal_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows package signal diff --git a/src/os/stat_nacl.go b/src/os/stat_nacljs.go similarity index 98% rename from src/os/stat_nacl.go rename to src/os/stat_nacljs.go index 0c53f2faa4378..f14add8674a50 100644 --- a/src/os/stat_nacl.go +++ b/src/os/stat_nacljs.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build js,wasm nacl + package os import ( diff --git a/src/os/stat_unix.go b/src/os/stat_unix.go index b58417150c489..856b49929f322 100644 --- a/src/os/stat_unix.go +++ b/src/os/stat_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package os diff --git a/src/os/sys_bsd.go b/src/os/sys_bsd.go index 78705c286d67f..d820be2ab6720 100644 --- a/src/os/sys_bsd.go +++ b/src/os/sys_bsd.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd nacl netbsd openbsd +// +build darwin dragonfly freebsd js,wasm nacl netbsd openbsd package os diff --git a/src/os/sys_js.go b/src/os/sys_js.go new file mode 100644 index 0000000000000..e860654f81204 --- /dev/null +++ b/src/os/sys_js.go @@ -0,0 +1,11 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package os + +// supportsCloseOnExec reports whether the platform supports the +// O_CLOEXEC flag. +const supportsCloseOnExec = false diff --git a/src/os/timeout_test.go b/src/os/timeout_test.go index 6105f9b1a1e95..1886accb55f7c 100644 --- a/src/os/timeout_test.go +++ b/src/os/timeout_test.go @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. // +build !nacl +// +build !js // +build !plan9 // +build !windows diff --git a/src/os/user/lookup_unix.go b/src/os/user/lookup_unix.go index 05f39be40b31c..c4e9ba1e81889 100644 --- a/src/os/user/lookup_unix.go +++ b/src/os/user/lookup_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd !android,linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm !android,linux nacl netbsd openbsd solaris // +build !cgo osusergo package user diff --git a/src/os/wait_unimp.go b/src/os/wait_unimp.go index b71e93f104a87..3d8210f5bdcee 100644 --- a/src/os/wait_unimp.go +++ b/src/os/wait_unimp.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly nacl netbsd openbsd solaris +// +build darwin dragonfly js,wasm nacl netbsd openbsd solaris package os diff --git a/src/path/filepath/path_unix.go b/src/path/filepath/path_unix.go index d77ff24cdc318..349dea7b53952 100644 --- a/src/path/filepath/path_unix.go +++ b/src/path/filepath/path_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package filepath diff --git a/src/runtime/debug/heapdump_test.go b/src/runtime/debug/heapdump_test.go index 7d5b950895d53..c986efcb325e7 100644 --- a/src/runtime/debug/heapdump_test.go +++ b/src/runtime/debug/heapdump_test.go @@ -13,8 +13,8 @@ import ( ) func TestWriteHeapDumpNonempty(t *testing.T) { - if runtime.GOOS == "nacl" { - t.Skip("WriteHeapDump is not available on NaCl.") + if runtime.GOOS == "nacl" || runtime.GOOS == "js" { + t.Skipf("WriteHeapDump is not available on %s.", runtime.GOOS) } f, err := ioutil.TempFile("", "heapdumptest") if err != nil { @@ -42,8 +42,8 @@ func objfin(x *Obj) { } func TestWriteHeapDumpFinalizers(t *testing.T) { - if runtime.GOOS == "nacl" { - t.Skip("WriteHeapDump is not available on NaCl.") + if runtime.GOOS == "nacl" || runtime.GOOS == "js" { + t.Skipf("WriteHeapDump is not available on %s.", runtime.GOOS) } f, err := ioutil.TempFile("", "heapdumptest") if err != nil { diff --git a/src/runtime/os_js.go b/src/runtime/os_js.go new file mode 100644 index 0000000000000..ad6db18b74793 --- /dev/null +++ b/src/runtime/os_js.go @@ -0,0 +1,145 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package runtime + +import ( + "unsafe" +) + +func exit(code int32) + +func write(fd uintptr, p unsafe.Pointer, n int32) int32 { + if fd > 2 { + throw("runtime.write to fd > 2 is unsupported") + } + wasmWrite(fd, p, n) + return n +} + +// Stubs so tests can link correctly. These should never be called. +func open(name *byte, mode, perm int32) int32 { panic("not implemented") } +func closefd(fd int32) int32 { panic("not implemented") } +func read(fd int32, p unsafe.Pointer, n int32) int32 { panic("not implemented") } + +//go:noescape +func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) + +func usleep(usec uint32) + +func exitThread(wait *uint32) + +type mOS struct{} + +func osyield() + +const _SIGSEGV = 0xb + +func sigpanic() { + g := getg() + if !canpanic(g) { + throw("unexpected signal during runtime execution") + } + + // js only invokes the exception handler for memory faults. + g.sig = _SIGSEGV + panicmem() +} + +type sigset struct{} + +// Called to initialize a new m (including the bootstrap m). +// Called on the parent thread (main thread in case of bootstrap), can allocate memory. +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) + mp.gsignal.m = mp +} + +//go:nosplit +func msigsave(mp *m) { +} + +//go:nosplit +func msigrestore(sigmask sigset) { +} + +//go:nosplit +//go:nowritebarrierrec +func clearSignalHandlers() { +} + +//go:nosplit +func sigblock() { +} + +// Called to initialize a new m (including the bootstrap m). +// Called on the new thread, cannot allocate memory. +func minit() { +} + +// Called from dropm to undo the effect of an minit. +func unminit() { +} + +func osinit() { + ncpu = 1 + getg().m.procid = 2 + physPageSize = 64 * 1024 +} + +// wasm has no signals +const _NSIG = 0 + +func signame(sig uint32) string { + return "" +} + +func crash() { + *(*int32)(nil) = 0 +} + +func getRandomData(r []byte) + +func goenvs() { + goenvs_unix() +} + +func initsig(preinit bool) { +} + +// May run with m.p==nil, so write barriers are not allowed. +//go:nowritebarrier +func newosproc(mp *m) { + panic("newosproc: not implemented") +} + +func setProcessCPUProfiler(hz int32) {} +func setThreadCPUProfiler(hz int32) {} +func sigdisable(uint32) {} +func sigenable(uint32) {} +func sigignore(uint32) {} + +//go:linkname os_sigpipe os.sigpipe +func os_sigpipe() { + throw("too many writes on closed pipe") +} + +//go:nosplit +func cputicks() int64 { + // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand(). + // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. + // TODO: need more entropy to better seed fastrand. + return nanotime() +} + +//go:linkname syscall_now syscall.now +func syscall_now() (sec int64, nsec int32) { + sec, nsec, _ = time_now() + return +} + +// gsignalStack is unused on js. +type gsignalStack struct{} diff --git a/src/runtime/sys_wasm.s b/src/runtime/sys_wasm.s index c9815821a6c43..9a67ceec63e76 100644 --- a/src/runtime/sys_wasm.s +++ b/src/runtime/sys_wasm.s @@ -193,3 +193,7 @@ TEXT ·nanotime(SB), NOSPLIT, $0 TEXT ·walltime(SB), NOSPLIT, $0 CallImport RET + +TEXT ·getRandomData(SB), NOSPLIT, $0 + CallImport + RET diff --git a/src/runtime/trace/trace_test.go b/src/runtime/trace/trace_test.go index f289bd6f85811..fc81abc30ff25 100644 --- a/src/runtime/trace/trace_test.go +++ b/src/runtime/trace/trace_test.go @@ -180,6 +180,9 @@ func testBrokenTimestamps(t *testing.T, data []byte) { } func TestTraceStress(t *testing.T) { + if runtime.GOOS == "js" { + t.Skip("no os.Pipe on js") + } if IsEnabled() { t.Skip("skipping because -test.trace is set") } @@ -322,6 +325,9 @@ func TestTraceStress(t *testing.T) { // Do a bunch of various stuff (timers, GC, network, etc) in a separate goroutine. // And concurrently with all that start/stop trace 3 times. func TestTraceStressStartStop(t *testing.T) { + if runtime.GOOS == "js" { + t.Skip("no os.Pipe on js") + } if IsEnabled() { t.Skip("skipping because -test.trace is set") } diff --git a/src/syscall/dirent.go b/src/syscall/dirent.go index 4db2d4355b5f4..26cbbbce2ad54 100644 --- a/src/syscall/dirent.go +++ b/src/syscall/dirent.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris package syscall diff --git a/src/syscall/endian_little.go b/src/syscall/endian_little.go index bd6f06e426779..013d878b8df9f 100644 --- a/src/syscall/endian_little.go +++ b/src/syscall/endian_little.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // -// +build 386 amd64 amd64p32 arm arm64 ppc64le mips64le mipsle +// +build 386 amd64 amd64p32 arm arm64 ppc64le mips64le mipsle wasm package syscall diff --git a/src/syscall/env_unix.go b/src/syscall/env_unix.go index 5bf3336ce5ae3..1ebc0b17f2be9 100644 --- a/src/syscall/env_unix.go +++ b/src/syscall/env_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris // Unix environment variables. diff --git a/src/syscall/fs_js.go b/src/syscall/fs_js.go new file mode 100644 index 0000000000000..141d46803c81e --- /dev/null +++ b/src/syscall/fs_js.go @@ -0,0 +1,497 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package syscall + +import ( + "io" + "sync" + "syscall/js" +) + +// Provided by package runtime. +func now() (sec int64, nsec int32) + +var jsProcess = js.Global.Get("process") +var jsFS = js.Global.Get("fs") +var constants = jsFS.Get("constants") + +var ( + nodeWRONLY = constants.Get("O_WRONLY").Int() + nodeRDWR = constants.Get("O_RDWR").Int() + nodeCREATE = constants.Get("O_CREAT").Int() + nodeTRUNC = constants.Get("O_TRUNC").Int() + nodeAPPEND = constants.Get("O_APPEND").Int() + nodeEXCL = constants.Get("O_EXCL").Int() + nodeNONBLOCK = constants.Get("O_NONBLOCK").Int() + nodeSYNC = constants.Get("O_SYNC").Int() +) + +type jsFile struct { + path string + entries []string + pos int64 + seeked bool +} + +var filesMu sync.Mutex +var files = map[int]*jsFile{ + 0: &jsFile{}, + 1: &jsFile{}, + 2: &jsFile{}, +} + +func fdToFile(fd int) (*jsFile, error) { + filesMu.Lock() + f, ok := files[fd] + filesMu.Unlock() + if !ok { + return nil, EBADF + } + return f, nil +} + +func Open(path string, openmode int, perm uint32) (int, error) { + if err := checkPath(path); err != nil { + return 0, err + } + + flags := 0 + if openmode&O_WRONLY != 0 { + flags |= nodeWRONLY + } + if openmode&O_RDWR != 0 { + flags |= nodeRDWR + } + if openmode&O_CREATE != 0 { + flags |= nodeCREATE + } + if openmode&O_TRUNC != 0 { + flags |= nodeTRUNC + } + if openmode&O_APPEND != 0 { + flags |= nodeAPPEND + } + if openmode&O_EXCL != 0 { + flags |= nodeEXCL + } + if openmode&O_NONBLOCK != 0 { + flags |= nodeNONBLOCK + } + if openmode&O_SYNC != 0 { + flags |= nodeSYNC + } + + jsFD, err := fsCall("openSync", path, flags, perm) + if err != nil { + return 0, err + } + fd := jsFD.Int() + + var entries []string + if stat, err := fsCall("fstatSync", fd); err == nil && stat.Call("isDirectory").Bool() { + dir, err := fsCall("readdirSync", path) + if err != nil { + return 0, err + } + entries = make([]string, dir.Length()) + for i := range entries { + entries[i] = dir.Index(i).String() + } + } + + f := &jsFile{ + path: path, + entries: entries, + } + filesMu.Lock() + files[fd] = f + filesMu.Unlock() + return fd, nil +} + +func Close(fd int) error { + filesMu.Lock() + delete(files, fd) + filesMu.Unlock() + _, err := fsCall("closeSync", fd) + return err +} + +func CloseOnExec(fd int) { + // nothing to do - no exec +} + +func Mkdir(path string, perm uint32) error { + if err := checkPath(path); err != nil { + return err + } + _, err := fsCall("mkdirSync", path, perm) + return err +} + +func ReadDirent(fd int, buf []byte) (int, error) { + f, err := fdToFile(fd) + if err != nil { + return 0, err + } + if f.entries == nil { + return 0, EINVAL + } + + n := 0 + for len(f.entries) > 0 { + entry := f.entries[0] + l := 2 + len(entry) + if l > len(buf) { + break + } + buf[0] = byte(l) + buf[1] = byte(l >> 8) + copy(buf[2:], entry) + buf = buf[l:] + n += l + f.entries = f.entries[1:] + } + + return n, nil +} + +func setStat(st *Stat_t, jsSt js.Value) { + st.Dev = int64(jsSt.Get("dev").Int()) + st.Ino = uint64(jsSt.Get("ino").Int()) + st.Mode = uint32(jsSt.Get("mode").Int()) + st.Nlink = uint32(jsSt.Get("nlink").Int()) + st.Uid = uint32(jsSt.Get("uid").Int()) + st.Gid = uint32(jsSt.Get("gid").Int()) + st.Rdev = int64(jsSt.Get("rdev").Int()) + st.Size = int64(jsSt.Get("size").Int()) + st.Blksize = int32(jsSt.Get("blksize").Int()) + st.Blocks = int32(jsSt.Get("blocks").Int()) + atime := int64(jsSt.Get("atimeMs").Int()) + st.Atime = atime / 1000 + st.AtimeNsec = (atime % 1000) * 1000000 + mtime := int64(jsSt.Get("mtimeMs").Int()) + st.Mtime = mtime / 1000 + st.MtimeNsec = (mtime % 1000) * 1000000 + ctime := int64(jsSt.Get("ctimeMs").Int()) + st.Ctime = ctime / 1000 + st.CtimeNsec = (ctime % 1000) * 1000000 +} + +func Stat(path string, st *Stat_t) error { + if err := checkPath(path); err != nil { + return err + } + jsSt, err := fsCall("statSync", path) + if err != nil { + return err + } + setStat(st, jsSt) + return nil +} + +func Lstat(path string, st *Stat_t) error { + if err := checkPath(path); err != nil { + return err + } + jsSt, err := fsCall("lstatSync", path) + if err != nil { + return err + } + setStat(st, jsSt) + return nil +} + +func Fstat(fd int, st *Stat_t) error { + jsSt, err := fsCall("fstatSync", fd) + if err != nil { + return err + } + setStat(st, jsSt) + return nil +} + +func Unlink(path string) error { + if err := checkPath(path); err != nil { + return err + } + _, err := fsCall("unlinkSync", path) + return err +} + +func Rmdir(path string) error { + if err := checkPath(path); err != nil { + return err + } + _, err := fsCall("rmdirSync", path) + return err +} + +func Chmod(path string, mode uint32) error { + if err := checkPath(path); err != nil { + return err + } + _, err := fsCall("chmodSync", path, mode) + return err +} + +func Fchmod(fd int, mode uint32) error { + _, err := fsCall("fchmodSync", fd, mode) + return err +} + +func Chown(path string, uid, gid int) error { + if err := checkPath(path); err != nil { + return err + } + return ENOSYS +} + +func Fchown(fd int, uid, gid int) error { + return ENOSYS +} + +func Lchown(path string, uid, gid int) error { + if err := checkPath(path); err != nil { + return err + } + return ENOSYS +} + +func UtimesNano(path string, ts []Timespec) error { + if err := checkPath(path); err != nil { + return err + } + if len(ts) != 2 { + return EINVAL + } + atime := ts[0].Sec + mtime := ts[1].Sec + _, err := fsCall("utimesSync", path, atime, mtime) + return err +} + +func Rename(from, to string) error { + if err := checkPath(from); err != nil { + return err + } + if err := checkPath(to); err != nil { + return err + } + _, err := fsCall("renameSync", from, to) + return err +} + +func Truncate(path string, length int64) error { + if err := checkPath(path); err != nil { + return err + } + _, err := fsCall("truncateSync", path, length) + return err +} + +func Ftruncate(fd int, length int64) error { + _, err := fsCall("ftruncateSync", fd, length) + return err +} + +func Getcwd(buf []byte) (n int, err error) { + defer recoverErr(&err) + cwd := jsProcess.Call("cwd").String() + n = copy(buf, cwd) + return n, nil +} + +func Chdir(path string) (err error) { + if err := checkPath(path); err != nil { + return err + } + defer recoverErr(&err) + jsProcess.Call("chdir", path) + return +} + +func Fchdir(fd int) error { + f, err := fdToFile(fd) + if err != nil { + return err + } + return Chdir(f.path) +} + +func Readlink(path string, buf []byte) (n int, err error) { + if err := checkPath(path); err != nil { + return 0, err + } + dst, err := fsCall("readlinkSync", path) + if err != nil { + return 0, err + } + n = copy(buf, dst.String()) + return n, nil +} + +func Link(path, link string) error { + if err := checkPath(path); err != nil { + return err + } + if err := checkPath(link); err != nil { + return err + } + _, err := fsCall("linkSync", path, link) + return err +} + +func Symlink(path, link string) error { + if err := checkPath(path); err != nil { + return err + } + if err := checkPath(link); err != nil { + return err + } + _, err := fsCall("symlinkSync", path, link) + return err +} + +func Fsync(fd int) error { + _, err := fsCall("fsyncSync", fd) + return err +} + +func Read(fd int, b []byte) (int, error) { + f, err := fdToFile(fd) + if err != nil { + return 0, err + } + + if f.seeked { + n, err := Pread(fd, b, f.pos) + f.pos += int64(n) + return n, err + } + + n, err := fsCall("readSync", fd, b, 0, len(b)) + if err != nil { + return 0, err + } + n2 := n.Int() + f.pos += int64(n2) + return n2, err +} + +func Write(fd int, b []byte) (int, error) { + f, err := fdToFile(fd) + if err != nil { + return 0, err + } + + if f.seeked { + n, err := Pwrite(fd, b, f.pos) + f.pos += int64(n) + return n, err + } + + n, err := fsCall("writeSync", fd, b, 0, len(b)) + if err != nil { + return 0, err + } + n2 := n.Int() + f.pos += int64(n2) + return n2, err +} + +func Pread(fd int, b []byte, offset int64) (int, error) { + n, err := fsCall("readSync", fd, b, 0, len(b), offset) + if err != nil { + return 0, err + } + return n.Int(), nil +} + +func Pwrite(fd int, b []byte, offset int64) (int, error) { + n, err := fsCall("writeSync", fd, b, 0, len(b), offset) + if err != nil { + return 0, err + } + return n.Int(), nil +} + +func Seek(fd int, offset int64, whence int) (int64, error) { + f, err := fdToFile(fd) + if err != nil { + return 0, err + } + + var newPos int64 + switch whence { + case io.SeekStart: + newPos = offset + case io.SeekCurrent: + newPos = f.pos + offset + case io.SeekEnd: + var st Stat_t + if err := Fstat(fd, &st); err != nil { + return 0, err + } + newPos = st.Size + offset + default: + return 0, errnoErr(EINVAL) + } + + if newPos < 0 { + return 0, errnoErr(EINVAL) + } + + f.seeked = true + f.pos = newPos + return newPos, nil +} + +func Dup(fd int) (int, error) { + return 0, ENOSYS +} + +func Dup2(fd, newfd int) error { + return ENOSYS +} + +func Pipe(fd []int) error { + return ENOSYS +} + +func fsCall(name string, args ...interface{}) (res js.Value, err error) { + defer recoverErr(&err) + res = jsFS.Call(name, args...) + return +} + +// checkPath checks that the path is not empty and that it contains no null characters. +func checkPath(path string) error { + if path == "" { + return EINVAL + } + for i := 0; i < len(path); i++ { + if path[i] == '\x00' { + return EINVAL + } + } + return nil +} + +func recoverErr(errPtr *error) { + if err := recover(); err != nil { + jsErr, ok := err.(js.Error) + if !ok { + panic(err) + } + errno, ok := errnoByCode[jsErr.Get("code").String()] + if !ok { + panic(err) + } + *errPtr = errnoErr(Errno(errno)) + } +} diff --git a/src/syscall/syscall_js.go b/src/syscall/syscall_js.go new file mode 100644 index 0000000000000..356d925462ca0 --- /dev/null +++ b/src/syscall/syscall_js.go @@ -0,0 +1,308 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js,wasm + +package syscall + +import ( + "sync" + "unsafe" +) + +const direntSize = 8 + 8 + 2 + 256 + +type Dirent struct { + Reclen uint16 + Name [256]byte +} + +func direntIno(buf []byte) (uint64, bool) { + return 1, true +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(Dirent{}.Reclen), unsafe.Sizeof(Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + reclen, ok := direntReclen(buf) + if !ok { + return 0, false + } + return reclen - uint64(unsafe.Offsetof(Dirent{}.Name)), true +} + +const PathMax = 256 + +// An Errno is an unsigned number describing an error condition. +// It implements the error interface. The zero Errno is by convention +// a non-error, so code to convert from Errno to error should use: +// err = nil +// if errno != 0 { +// err = errno +// } +type Errno uintptr + +func (e Errno) Error() string { + if 0 <= int(e) && int(e) < len(errorstr) { + s := errorstr[e] + if s != "" { + return s + } + } + return "errno " + itoa(int(e)) +} + +func (e Errno) Temporary() bool { + return e == EINTR || e == EMFILE || e.Timeout() +} + +func (e Errno) Timeout() bool { + return e == EAGAIN || e == EWOULDBLOCK || e == ETIMEDOUT +} + +// A Signal is a number describing a process signal. +// It implements the os.Signal interface. +type Signal int + +const ( + _ Signal = iota + SIGCHLD + SIGINT + SIGKILL + SIGTRAP + SIGQUIT +) + +func (s Signal) Signal() {} + +func (s Signal) String() string { + if 0 <= s && int(s) < len(signals) { + str := signals[s] + if str != "" { + return str + } + } + return "signal " + itoa(int(s)) +} + +var signals = [...]string{} + +// File system + +const ( + Stdin = 0 + Stdout = 1 + Stderr = 2 +) + +const ( + O_RDONLY = 0 + O_WRONLY = 1 + O_RDWR = 2 + + O_CREAT = 0100 + O_CREATE = O_CREAT + O_TRUNC = 01000 + O_APPEND = 02000 + O_EXCL = 0200 + O_NONBLOCK = 04000 + O_SYNC = 010000 + + O_CLOEXEC = 0 +) + +const ( + F_DUPFD = 0 + F_GETFD = 1 + F_SETFD = 2 + F_GETFL = 3 + F_SETFL = 4 + F_GETOWN = 5 + F_SETOWN = 6 + F_GETLK = 7 + F_SETLK = 8 + F_SETLKW = 9 + F_RGETLK = 10 + F_RSETLK = 11 + F_CNVT = 12 + F_RSETLKW = 13 + + F_RDLCK = 1 + F_WRLCK = 2 + F_UNLCK = 3 + F_UNLKSYS = 4 +) + +const ( + S_IFMT = 0000370000 + S_IFSHM_SYSV = 0000300000 + S_IFSEMA = 0000270000 + S_IFCOND = 0000260000 + S_IFMUTEX = 0000250000 + S_IFSHM = 0000240000 + S_IFBOUNDSOCK = 0000230000 + S_IFSOCKADDR = 0000220000 + S_IFDSOCK = 0000210000 + + S_IFSOCK = 0000140000 + S_IFLNK = 0000120000 + S_IFREG = 0000100000 + S_IFBLK = 0000060000 + S_IFDIR = 0000040000 + S_IFCHR = 0000020000 + S_IFIFO = 0000010000 + + S_UNSUP = 0000370000 + + S_ISUID = 0004000 + S_ISGID = 0002000 + S_ISVTX = 0001000 + + S_IREAD = 0400 + S_IWRITE = 0200 + S_IEXEC = 0100 + + S_IRWXU = 0700 + S_IRUSR = 0400 + S_IWUSR = 0200 + S_IXUSR = 0100 + + S_IRWXG = 070 + S_IRGRP = 040 + S_IWGRP = 020 + S_IXGRP = 010 + + S_IRWXO = 07 + S_IROTH = 04 + S_IWOTH = 02 + S_IXOTH = 01 +) + +type Stat_t struct { + Dev int64 + Ino uint64 + Mode uint32 + Nlink uint32 + Uid uint32 + Gid uint32 + Rdev int64 + Size int64 + Blksize int32 + Blocks int32 + Atime int64 + AtimeNsec int64 + Mtime int64 + MtimeNsec int64 + Ctime int64 + CtimeNsec int64 +} + +// Processes +// Not supported - just enough for package os. + +var ForkLock sync.RWMutex + +type WaitStatus uint32 + +func (w WaitStatus) Exited() bool { return false } +func (w WaitStatus) ExitStatus() int { return 0 } +func (w WaitStatus) Signaled() bool { return false } +func (w WaitStatus) Signal() Signal { return 0 } +func (w WaitStatus) CoreDump() bool { return false } +func (w WaitStatus) Stopped() bool { return false } +func (w WaitStatus) Continued() bool { return false } +func (w WaitStatus) StopSignal() Signal { return 0 } +func (w WaitStatus) TrapCause() int { return 0 } + +// XXX made up +type Rusage struct { + Utime Timeval + Stime Timeval +} + +// XXX made up +type ProcAttr struct { + Dir string + Env []string + Files []uintptr + Sys *SysProcAttr +} + +type SysProcAttr struct { +} + +func Syscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + return 0, 0, ENOSYS +} + +func Syscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + return 0, 0, ENOSYS +} + +func RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + return 0, 0, ENOSYS +} + +func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + return 0, 0, ENOSYS +} + +func Sysctl(key string) (string, error) { + if key == "kern.hostname" { + return "js", nil + } + return "", ENOSYS +} + +const ImplementsGetwd = true + +func Getwd() (wd string, err error) { + var buf [PathMax]byte + n, err := Getcwd(buf[0:]) + if err != nil { + return "", err + } + return string(buf[:n]), nil +} + +func Getegid() int { return 1 } +func Geteuid() int { return 1 } +func Getgid() int { return 1 } +func Getgroups() ([]int, error) { return []int{1}, nil } +func Getppid() int { return 2 } +func Getpid() int { return 3 } +func Gettimeofday(tv *Timeval) error { return ENOSYS } +func Getuid() int { return 1 } +func Kill(pid int, signum Signal) error { return ENOSYS } +func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) { + return 0, ENOSYS +} +func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle uintptr, err error) { + return 0, 0, ENOSYS +} +func Wait4(pid int, wstatus *WaitStatus, options int, rusage *Rusage) (wpid int, err error) { + return 0, ENOSYS +} + +type Iovec struct{} // dummy + +type Timespec struct { + Sec int64 + Nsec int64 +} + +type Timeval struct { + Sec int64 + Usec int64 +} + +func setTimespec(sec, nsec int64) Timespec { + return Timespec{Sec: sec, Nsec: nsec} +} + +func setTimeval(sec, usec int64) Timeval { + return Timeval{Sec: sec, Usec: usec} +} diff --git a/src/syscall/tables_nacljs.go b/src/syscall/tables_nacljs.go index e5c51c9c8990c..1c265f25c7abc 100644 --- a/src/syscall/tables_nacljs.go +++ b/src/syscall/tables_nacljs.go @@ -2,10 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl +// +build nacl js,wasm package syscall +import "runtime" + // TODO: generate with runtime/mknacl.sh, allow override with IRT. const ( sys_null = 1 @@ -254,7 +256,7 @@ var errorstr = [...]string{ EMLINK: "Too many links", EPIPE: "Broken pipe", ENAMETOOLONG: "File name too long", - ENOSYS: "not implemented on Native Client", + ENOSYS: "not implemented on " + runtime.GOOS, EDQUOT: "Quota exceeded", EDOM: "Math arg out of domain of func", ERANGE: "Math result not representable", @@ -361,3 +363,128 @@ func errnoErr(e Errno) error { } return e } + +var errnoByCode = map[string]Errno{ + "EPERM": EPERM, + "ENOENT": ENOENT, + "ESRCH": ESRCH, + "EINTR": EINTR, + "EIO": EIO, + "ENXIO": ENXIO, + "E2BIG": E2BIG, + "ENOEXEC": ENOEXEC, + "EBADF": EBADF, + "ECHILD": ECHILD, + "EAGAIN": EAGAIN, + "ENOMEM": ENOMEM, + "EACCES": EACCES, + "EFAULT": EFAULT, + "EBUSY": EBUSY, + "EEXIST": EEXIST, + "EXDEV": EXDEV, + "ENODEV": ENODEV, + "ENOTDIR": ENOTDIR, + "EISDIR": EISDIR, + "EINVAL": EINVAL, + "ENFILE": ENFILE, + "EMFILE": EMFILE, + "ENOTTY": ENOTTY, + "EFBIG": EFBIG, + "ENOSPC": ENOSPC, + "ESPIPE": ESPIPE, + "EROFS": EROFS, + "EMLINK": EMLINK, + "EPIPE": EPIPE, + "ENAMETOOLONG": ENAMETOOLONG, + "ENOSYS": ENOSYS, + "EDQUOT": EDQUOT, + "EDOM": EDOM, + "ERANGE": ERANGE, + "EDEADLK": EDEADLK, + "ENOLCK": ENOLCK, + "ENOTEMPTY": ENOTEMPTY, + "ELOOP": ELOOP, + "ENOMSG": ENOMSG, + "EIDRM": EIDRM, + "ECHRNG": ECHRNG, + "EL2NSYNC": EL2NSYNC, + "EL3HLT": EL3HLT, + "EL3RST": EL3RST, + "ELNRNG": ELNRNG, + "EUNATCH": EUNATCH, + "ENOCSI": ENOCSI, + "EL2HLT": EL2HLT, + "EBADE": EBADE, + "EBADR": EBADR, + "EXFULL": EXFULL, + "ENOANO": ENOANO, + "EBADRQC": EBADRQC, + "EBADSLT": EBADSLT, + "EDEADLOCK": EDEADLOCK, + "EBFONT": EBFONT, + "ENOSTR": ENOSTR, + "ENODATA": ENODATA, + "ETIME": ETIME, + "ENOSR": ENOSR, + "ENONET": ENONET, + "ENOPKG": ENOPKG, + "EREMOTE": EREMOTE, + "ENOLINK": ENOLINK, + "EADV": EADV, + "ESRMNT": ESRMNT, + "ECOMM": ECOMM, + "EPROTO": EPROTO, + "EMULTIHOP": EMULTIHOP, + "EDOTDOT": EDOTDOT, + "EBADMSG": EBADMSG, + "EOVERFLOW": EOVERFLOW, + "ENOTUNIQ": ENOTUNIQ, + "EBADFD": EBADFD, + "EREMCHG": EREMCHG, + "ELIBACC": ELIBACC, + "ELIBBAD": ELIBBAD, + "ELIBSCN": ELIBSCN, + "ELIBMAX": ELIBMAX, + "ELIBEXEC": ELIBEXEC, + "EILSEQ": EILSEQ, + "EUSERS": EUSERS, + "ENOTSOCK": ENOTSOCK, + "EDESTADDRREQ": EDESTADDRREQ, + "EMSGSIZE": EMSGSIZE, + "EPROTOTYPE": EPROTOTYPE, + "ENOPROTOOPT": ENOPROTOOPT, + "EPROTONOSUPPORT": EPROTONOSUPPORT, + "ESOCKTNOSUPPORT": ESOCKTNOSUPPORT, + "EOPNOTSUPP": EOPNOTSUPP, + "EPFNOSUPPORT": EPFNOSUPPORT, + "EAFNOSUPPORT": EAFNOSUPPORT, + "EADDRINUSE": EADDRINUSE, + "EADDRNOTAVAIL": EADDRNOTAVAIL, + "ENETDOWN": ENETDOWN, + "ENETUNREACH": ENETUNREACH, + "ENETRESET": ENETRESET, + "ECONNABORTED": ECONNABORTED, + "ECONNRESET": ECONNRESET, + "ENOBUFS": ENOBUFS, + "EISCONN": EISCONN, + "ENOTCONN": ENOTCONN, + "ESHUTDOWN": ESHUTDOWN, + "ETOOMANYREFS": ETOOMANYREFS, + "ETIMEDOUT": ETIMEDOUT, + "ECONNREFUSED": ECONNREFUSED, + "EHOSTDOWN": EHOSTDOWN, + "EHOSTUNREACH": EHOSTUNREACH, + "EALREADY": EALREADY, + "EINPROGRESS": EINPROGRESS, + "ESTALE": ESTALE, + "ENOTSUP": ENOTSUP, + "ENOMEDIUM": ENOMEDIUM, + "ECANCELED": ECANCELED, + "ELBIN": ELBIN, + "EFTYPE": EFTYPE, + "ENMFILE": ENMFILE, + "EPROCLIM": EPROCLIM, + "ENOSHARE": ENOSHARE, + "ECASECLASH": ECASECLASH, + "EWOULDBLOCK": EWOULDBLOCK, +} From 3d6e4ec0a8c2ef47211519b21b020131c0434003 Mon Sep 17 00:00:00 2001 From: Ben Shi Date: Sat, 26 May 2018 12:43:16 +0000 Subject: [PATCH 065/187] cmd/internal/obj/arm64: fix two issues in the assembler There are two issues in the arm64 assembler. 1. "CMPW $0x22220000, RSP" is encoded to 5b44a4d2ff031b6b, which is the combination of "MOVD $0x22220000, Rtmp" and "NEGSW Rtmp, ZR". The right encoding should be a combination of "MOVD $0x22220000, Rtmp" and "CMPW Rtmp, RSP". 2. "AND $0x22220000, R2, RSP" is encoded to 5b44a4d25f601b00, which is the combination of "MOVD $0x22220000, Rtmp" and an illegal instruction. The right behavior should be an error report of "illegal combination", since "AND Rtmp, RSP, RSP" is invalid in armv8. This CL fixes the above 2 issues and adds more test cases. fixes #25557 Change-Id: Ia510be26b58a229f5dfe8a5fa0b35569b2d566e7 Reviewed-on: https://go-review.googlesource.com/114796 Run-TryBot: Ben Shi TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-by: Cherry Zhang --- src/cmd/asm/internal/asm/testdata/arm64.s | 21 +++++++++++++++++++ .../asm/internal/asm/testdata/arm64error.s | 2 ++ src/cmd/internal/obj/arm64/asm7.go | 6 +++--- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/cmd/asm/internal/asm/testdata/arm64.s b/src/cmd/asm/internal/asm/testdata/arm64.s index ce6282f0ddb9a..54be761c543ce 100644 --- a/src/cmd/asm/internal/asm/testdata/arm64.s +++ b/src/cmd/asm/internal/asm/testdata/arm64.s @@ -190,6 +190,15 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 EOR $(1<<63), R1 // EOR $-9223372036854775808, R1 // 210041d2 EOR $(1<<63-1), R1 // EOR $9223372036854775807, R1 // 21f840d2 + AND $0x22220000, R3, R4 // AND $572653568, R3, R4 // 5b44a4d264001b8a + ORR $0x22220000, R3, R4 // ORR $572653568, R3, R4 // 5b44a4d264001baa + EOR $0x22220000, R3, R4 // EOR $572653568, R3, R4 // 5b44a4d264001bca + BIC $0x22220000, R3, R4 // BIC $572653568, R3, R4 // 5b44a4d264003b8a + ORN $0x22220000, R3, R4 // ORN $572653568, R3, R4 // 5b44a4d264003baa + EON $0x22220000, R3, R4 // EON $572653568, R3, R4 // 5b44a4d264003bca + ANDS $0x22220000, R3, R4 // ANDS $572653568, R3, R4 // 5b44a4d264001bea + BICS $0x22220000, R3, R4 // BICS $572653568, R3, R4 // 5b44a4d264003bea + AND $8, R0, RSP // 1f007d92 ORR $8, R0, RSP // 1f007db2 EOR $8, R0, RSP // 1f007dd2 @@ -390,6 +399,18 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 CMP R1>>22, R2 CMP R1<<33, R2 CMP R22.SXTX, RSP // ffe336eb + + CMP $0x22220000, RSP // CMP $572653568, RSP // 5b44a4d2ff633beb + CMPW $0x22220000, RSP // CMPW $572653568, RSP // 5b44a4d2ff633b6b + +// TST + TST $15, R2 // 5f0c40f2 + TST R1, R2 // 5f0001ea + TST R1->11, R2 // 5f2c81ea + TST R1>>22, R2 // 5f5841ea + TST R1<<33, R2 // 5f8401ea + TST $0x22220000, R3 // TST $572653568, R3 // 5b44a4d27f001bea + // // CBZ // diff --git a/src/cmd/asm/internal/asm/testdata/arm64error.s b/src/cmd/asm/internal/asm/testdata/arm64error.s index 36829686f6231..be2251e442e0c 100644 --- a/src/cmd/asm/internal/asm/testdata/arm64error.s +++ b/src/cmd/asm/internal/asm/testdata/arm64error.s @@ -87,4 +87,6 @@ TEXT errors(SB),$0 MADD R1, R2, R3 // ERROR "illegal combination" CINC CS, R2, R3, R4 // ERROR "illegal combination" CSEL LT, R1, R2 // ERROR "illegal combination" + AND $0x22220000, R2, RSP // ERROR "illegal combination" + ANDS $0x22220000, R2, RSP // ERROR "illegal combination" RET diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index bf96bb58a6abb..e7271437573ef 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -245,12 +245,12 @@ var optab = []Optab{ {AANDS, C_BITCON, C_REG, C_NONE, C_REG, 53, 4, 0, 0, 0}, {AANDS, C_BITCON, C_NONE, C_NONE, C_REG, 53, 4, 0, 0, 0}, {ATST, C_BITCON, C_REG, C_NONE, C_NONE, 53, 4, 0, 0, 0}, - {AAND, C_MOVCON, C_REG, C_NONE, C_RSP, 62, 8, 0, 0, 0}, + {AAND, C_MOVCON, C_REG, C_NONE, C_REG, 62, 8, 0, 0, 0}, {AAND, C_MOVCON, C_NONE, C_NONE, C_REG, 62, 8, 0, 0, 0}, {AANDS, C_MOVCON, C_REG, C_NONE, C_REG, 62, 8, 0, 0, 0}, {AANDS, C_MOVCON, C_NONE, C_NONE, C_REG, 62, 8, 0, 0, 0}, {ATST, C_MOVCON, C_REG, C_NONE, C_NONE, 62, 8, 0, 0, 0}, - {AAND, C_VCON, C_REG, C_NONE, C_RSP, 28, 8, 0, LFROM, 0}, + {AAND, C_VCON, C_REG, C_NONE, C_REG, 28, 8, 0, LFROM, 0}, {AAND, C_VCON, C_NONE, C_NONE, C_REG, 28, 8, 0, LFROM, 0}, {AANDS, C_VCON, C_REG, C_NONE, C_REG, 28, 8, 0, LFROM, 0}, {AANDS, C_VCON, C_NONE, C_NONE, C_REG, 28, 8, 0, LFROM, 0}, @@ -3548,7 +3548,7 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) { if r == 0 { r = rt } - if p.To.Type != obj.TYPE_NONE && (p.To.Reg == REGSP || r == REGSP) { + if p.To.Reg == REGSP || r == REGSP { o2 = c.opxrrr(p, p.As, false) o2 |= REGTMP & 31 << 16 o2 |= LSL0_64 From 6cfeedb229bcad20b99fafe3d127a087edcd07e5 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 31 May 2018 15:18:21 -0700 Subject: [PATCH 066/187] go/types: remove visited flag for constants and variables (cleanup) Now that we have a color marking scheme for all objects, the pre-existing 'visited' flag for constants and variables is redundant: visited is the same as marking an object non-white. Refactor the respective 'visited' flag logic from constDecl and varDecl into the color switch in objDecl and remove the 'visited' flag. Follow-up on https://go-review.googlesource.com/c/go/+/114517 . Change-Id: Ie20de65e3b26a5a6ff7b0eddc3d089f56be204e8 Reviewed-on: https://go-review.googlesource.com/115619 Reviewed-by: Alan Donovan --- src/go/types/decl.go | 49 ++++++++++++++---------------------------- src/go/types/object.go | 6 ++---- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/src/go/types/decl.go b/src/go/types/decl.go index 9a27fbbed6f0c..8430ebddb77c0 100644 --- a/src/go/types/decl.go +++ b/src/go/types/decl.go @@ -113,27 +113,29 @@ func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { case grey: // We have a cycle. // In the existing code, this is marked by a non-nil type - // for the object except for constants and variables, which - // have their own "visited" flag (the new marking approach - // will allow us to remove that flag eventually). Their type - // may be nil because they haven't determined their init - // values yet (from which to deduce the type). But in that - // case, they must have been marked as visited. - // For now, handle constants and variables specially. - visited := false + // for the object except for constants and variables whose + // type may be non-nil (known), or nil if it depends on the + // not-yet known initialization value. + // In the former case, set the type to Typ[Invalid] because + // we have an initialization cycle. The cycle error will be + // reported later, when determining initialization order. + // TODO(gri) Report cycle here and simplify initialization + // order code. switch obj := obj.(type) { case *Const: - visited = obj.visited + if obj.typ == nil { + obj.typ = Typ[Invalid] + } case *Var: - visited = obj.visited + if obj.typ == nil { + obj.typ = Typ[Invalid] + } case *TypeName: - assert(obj.Type() != nil) if useCycleMarking { check.typeCycle(obj) } - return case *Func: // Cycles involving functions require variables in @@ -142,19 +144,12 @@ func (check *Checker) objDecl(obj Object, def *Named, path []*TypeName) { // function type is set to an empty signature which // makes it impossible to initialize a variable with // the function). - assert(obj.Type() != nil) - return default: unreachable() } - - // we have a *Const or *Var - if obj.Type() != nil { - return - } - assert(visited) - + assert(obj.Type() != nil) + return } if trace { @@ -260,12 +255,6 @@ func (check *Checker) typeCycle(obj *TypeName) { func (check *Checker) constDecl(obj *Const, typ, init ast.Expr) { assert(obj.typ == nil) - if obj.visited { - obj.typ = Typ[Invalid] - return - } - obj.visited = true - // use the correct value of iota check.iota = obj.val defer func() { check.iota = nil }() @@ -299,12 +288,6 @@ func (check *Checker) constDecl(obj *Const, typ, init ast.Expr) { func (check *Checker) varDecl(obj *Var, lhs []*Var, typ, init ast.Expr) { assert(obj.typ == nil) - if obj.visited { - obj.typ = Typ[Invalid] - return - } - obj.visited = true - // determine type, if any if typ != nil { obj.typ = check.typ(typ) diff --git a/src/go/types/object.go b/src/go/types/object.go index 1305a9db6eb98..07adfbc34c390 100644 --- a/src/go/types/object.go +++ b/src/go/types/object.go @@ -197,14 +197,13 @@ func (obj *PkgName) Imported() *Package { return obj.imported } // A Const represents a declared constant. type Const struct { object - val constant.Value - visited bool // for initialization cycle detection + val constant.Value } // NewConst returns a new constant with value val. // The remaining arguments set the attributes found with all Objects. func NewConst(pos token.Pos, pkg *Package, name string, typ Type, val constant.Value) *Const { - return &Const{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, val, false} + return &Const{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, val} } // Val returns the constant's value. @@ -256,7 +255,6 @@ func (obj *TypeName) IsAlias() bool { type Var struct { object embedded bool // if set, the variable is an embedded struct field, and name is the type name - visited bool // for initialization cycle detection isField bool // var is struct field used bool // set if the variable was used } From b280edb89eae1974e3f218726814b03b3558e005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20M=C3=B6hrmann?= Date: Fri, 1 Jun 2018 19:43:58 +0200 Subject: [PATCH 067/187] cmd/compile: fix comment to reference runtime.countrunes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates #24923 Change-Id: Ie5a1b54b023381b58df618080f3d742a50d46d8b Reviewed-on: https://go-review.googlesource.com/115836 Reviewed-by: Josh Bleecher Snyder Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/walk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index f42d1e43db6ea..591c8f3bfecbc 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -4078,7 +4078,7 @@ func canMergeLoads() bool { } // isRuneCount reports whether n is of the form len([]rune(string)). -// These are optimized into a call to runtime.runecount. +// These are optimized into a call to runtime.countrunes. func isRuneCount(n *Node) bool { return Debug['N'] == 0 && !instrumenting && n.Op == OLEN && n.Left.Op == OSTRARRAYRUNE } From c7519f0daee3289e9877266af21fe69905fd0d9c Mon Sep 17 00:00:00 2001 From: Hana Kim Date: Thu, 31 May 2018 16:14:04 -0400 Subject: [PATCH 068/187] runtime/trace: remove remaining NewContext reference This is a documentation error. Change-Id: I083021f151f7e80a0b9083b98452ae1f5920640d Reviewed-on: https://go-review.googlesource.com/115598 Reviewed-by: Peter Weinberger --- src/runtime/trace/trace.go | 43 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/runtime/trace/trace.go b/src/runtime/trace/trace.go index f3ea312d277bc..b6a594355a9f8 100644 --- a/src/runtime/trace/trace.go +++ b/src/runtime/trace/trace.go @@ -70,7 +70,7 @@ // operations such as an RPC request, an HTTP request, or an // interesting local operation which may require multiple goroutines // working together. Since tasks can involve multiple goroutines, -// they are tracked via a context.Context object. NewContext creates +// they are tracked via a context.Context object. NewTask creates // a new task and embeds it in the returned context.Context object. // Log messages and regions are attached to the task, if any, in the // Context passed to Log and WithRegion. @@ -80,26 +80,27 @@ // the trace tool can identify the goroutines involved in a specific // cappuccino order. // -// ctx, taskEnd:= trace.NewContext(ctx, "makeCappuccino") -// trace.Log(ctx, "orderID", orderID) -// -// milk := make(chan bool) -// espresso := make(chan bool) -// -// go func() { -// trace.WithRegion(ctx, "steamMilk", steamMilk) -// milk<-true -// })() -// go func() { -// trace.WithRegion(ctx, "extractCoffee", extractCoffee) -// espresso<-true -// })() -// go func() { -// defer taskEnd() // When assemble is done, the order is complete. -// <-espresso -// <-milk -// trace.WithRegion(ctx, "mixMilkCoffee", mixMilkCoffee) -// })() +// ctx, task := trace.NewTask(ctx, "makeCappuccino") +// trace.Log(ctx, "orderID", orderID) + +// milk := make(chan bool) +// espresso := make(chan bool) + +// go func() { +// trace.WithRegion(ctx, "steamMilk", steamMilk) +// milk <- true +// }() +// go func() { +// trace.WithRegion(ctx, "extractCoffee", extractCoffee) +// espresso <- true +// }() +// go func() { +// defer task.End() // When assemble is done, the order is complete. +// <-espresso +// <-milk +// trace.WithRegion(ctx, "mixMilkCoffee", mixMilkCoffee) +// }() +// // // The trace tool computes the latency of a task by measuring the // time between the task creation and the task end and provides From 7cb1810fe8117d4c5112ecea7a65f28f03009ef7 Mon Sep 17 00:00:00 2001 From: Yury Smolsky Date: Fri, 1 Jun 2018 23:42:23 +0300 Subject: [PATCH 069/187] test: skip test/fixedbugs/bug345.go on windows Before the CL 115277 we did not run the test on Windows, so let's just go back to not running the test on Windows. There is nothing OS-specific about this test, so skipping it on Windows doesn't seem like a big deal. Updates #25693 Fixes #25586 Change-Id: I1eb3e158b322d73e271ef388f8c6e2f2af0a0729 Reviewed-on: https://go-review.googlesource.com/115857 Run-TryBot: Yury Smolsky TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- test/fixedbugs/bug345.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/fixedbugs/bug345.go b/test/fixedbugs/bug345.go index 917592118dee0..b974a61ffb2b3 100644 --- a/test/fixedbugs/bug345.go +++ b/test/fixedbugs/bug345.go @@ -1,8 +1,10 @@ +// +build !windows // errorcheckdir -n -// run // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package ignored + +// TODO(ysmolsky): Fix golang.org/issue/25693 to enable on Windows. From 161874da2ab6d5372043a1f3938a81a19d1165ad Mon Sep 17 00:00:00 2001 From: Tim Cooper Date: Fri, 1 Jun 2018 17:29:59 -0300 Subject: [PATCH 070/187] all: update comment URLs from HTTP to HTTPS, where possible Each URL was manually verified to ensure it did not serve up incorrect content. Change-Id: I4dc846227af95a73ee9a3074d0c379ff0fa955df Reviewed-on: https://go-review.googlesource.com/115798 Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor --- src/archive/tar/format.go | 2 +- src/archive/zip/struct.go | 4 ++-- src/cmd/cgo/out.go | 2 +- src/cmd/compile/internal/gc/phi.go | 2 +- src/cmd/compile/internal/ssa/sparsemap.go | 2 +- src/cmd/compile/internal/ssa/sparseset.go | 2 +- src/cmd/dist/sys_windows.go | 2 +- src/cmd/go/go_test.go | 2 +- src/cmd/internal/obj/s390x/a.out.go | 2 +- src/cmd/link/internal/amd64/asm.go | 2 +- src/cmd/link/internal/ld/lib.go | 2 +- src/cmd/link/internal/ld/macho.go | 2 +- src/cmd/link/internal/ld/pe.go | 2 +- src/cmd/link/internal/loadelf/ldelf.go | 2 +- src/cmd/link/internal/wasm/asm.go | 2 +- src/compress/bzip2/bzip2.go | 2 +- src/compress/flate/deflate.go | 2 +- src/compress/lzw/reader_test.go | 2 +- src/compress/zlib/reader_test.go | 2 +- src/crypto/aes/aes_test.go | 2 +- src/crypto/aes/block.go | 4 ++-- src/crypto/aes/const.go | 2 +- src/crypto/cipher/cfb_test.go | 2 +- src/crypto/cipher/cipher.go | 2 +- src/crypto/cipher/gcm.go | 2 +- src/crypto/ecdsa/ecdsa_test.go | 2 +- src/crypto/elliptic/elliptic.go | 6 +++--- src/crypto/elliptic/p224.go | 4 ++-- src/crypto/elliptic/p256.go | 6 +++--- src/crypto/elliptic/p256_amd64.go | 2 +- src/crypto/elliptic/p256_asm_amd64.s | 2 +- src/crypto/elliptic/p256_asm_s390x.s | 10 +++++----- src/crypto/hmac/hmac.go | 2 +- src/crypto/hmac/hmac_test.go | 4 ++-- src/crypto/rc4/rc4_test.go | 4 ++-- src/crypto/rsa/rsa.go | 2 +- src/crypto/sha256/sha256block_386.s | 2 +- src/crypto/sha256/sha256block_amd64.s | 2 +- src/crypto/sha256/sha256block_ppc64le.s | 2 +- src/crypto/sha512/sha512block_amd64.s | 4 ++-- src/crypto/sha512/sha512block_ppc64le.s | 2 +- src/crypto/tls/common.go | 6 +++--- src/crypto/tls/conn.go | 4 ++-- src/crypto/tls/handshake_messages.go | 18 +++++++++--------- src/crypto/tls/key_agreement.go | 4 ++-- src/crypto/tls/prf.go | 2 +- src/crypto/x509/pkix/pkix.go | 2 +- src/crypto/x509/sha2_windows_test.go | 2 +- src/encoding/json/decode.go | 2 +- src/encoding/json/number_test.go | 2 +- src/encoding/xml/xml.go | 12 ++++++------ src/hash/crc32/crc32.go | 8 ++++---- src/hash/crc32/crc32_amd64.s | 2 +- src/hash/crc64/crc64.go | 2 +- src/html/template/attr.go | 8 ++++---- src/html/template/content.go | 4 ++-- src/html/template/context.go | 4 ++-- src/html/template/css.go | 12 ++++++------ src/html/template/escape.go | 4 ++-- src/html/template/html.go | 4 ++-- src/html/template/js.go | 2 +- src/html/template/js_test.go | 4 ++-- src/html/template/transition.go | 6 +++--- src/image/color/palette/gen.go | 2 +- src/image/color/palette/palette.go | 2 +- src/image/color/ycbcr.go | 6 +++--- src/image/gif/reader.go | 2 +- src/image/jpeg/reader.go | 10 +++++----- src/image/png/reader.go | 6 +++--- src/internal/poll/fd_windows.go | 6 +++--- src/math/all_test.go | 2 +- src/math/big/float.go | 2 +- src/math/big/float_test.go | 4 ++-- src/math/big/floatconv_test.go | 4 ++-- src/math/big/prime.go | 4 ++-- src/math/big/prime_test.go | 4 ++-- src/math/big/ratconv_test.go | 4 ++-- src/math/cmplx/cmath_test.go | 2 +- src/math/erfinv.go | 2 +- src/math/exp_amd64.s | 2 +- src/math/rand/exp.go | 2 +- src/mime/multipart/multipart.go | 4 ++-- src/mime/quotedprintable/writer_test.go | 2 +- src/net/conf.go | 4 ++-- src/net/conf_test.go | 2 +- src/net/dnsconfig_unix.go | 2 +- src/net/http/cgi/child.go | 2 +- src/net/http/client.go | 2 +- src/net/http/cookie.go | 4 ++-- src/net/http/httputil/reverseproxy.go | 2 +- src/net/http/server.go | 2 +- src/net/http/sniff.go | 2 +- src/net/http/transport.go | 2 +- src/net/tcpsock_posix.go | 2 +- src/net/textproto/reader.go | 2 +- src/net/textproto/reader_test.go | 2 +- src/os/env_test.go | 2 +- src/os/file_unix.go | 2 +- src/os/sys_unix.go | 2 +- src/regexp/exec.go | 4 ++-- src/regexp/regexp.go | 4 ++-- src/regexp/syntax/compile.go | 2 +- src/runtime/defs3_linux.go | 2 +- src/runtime/internal/atomic/sys_linux_arm.s | 2 +- src/runtime/memmove_amd64.s | 2 +- src/runtime/mgclarge.go | 4 ++-- src/runtime/proc.go | 4 ++-- src/runtime/race/output_test.go | 2 +- src/runtime/race_amd64.s | 2 +- src/runtime/sema.go | 4 ++-- src/runtime/stubs.go | 2 +- src/runtime/sys_windows_386.s | 2 +- src/runtime/sys_windows_amd64.s | 2 +- src/runtime/vdso_linux.go | 2 +- src/runtime/vlop_arm_test.go | 2 +- src/sort/sort_test.go | 2 +- src/strconv/atof_test.go | 4 ++-- src/strconv/extfloat.go | 2 +- src/strconv/ftoa_test.go | 4 ++-- src/strings/search.go | 4 ++-- src/strings/search_test.go | 2 +- src/syscall/env_windows.go | 2 +- src/syscall/exec_windows.go | 2 +- src/syscall/route_freebsd_32bit.go | 2 +- src/syscall/security_windows.go | 2 +- src/syscall/types_windows.go | 2 +- src/time/zoneinfo_read.go | 2 +- src/time/zoneinfo_unix.go | 2 +- src/unicode/maketables.go | 4 ++-- src/unicode/utf16/utf16_test.go | 2 +- 130 files changed, 206 insertions(+), 206 deletions(-) diff --git a/src/archive/tar/format.go b/src/archive/tar/format.go index 6e29698a14a54..1f89d0c59a159 100644 --- a/src/archive/tar/format.go +++ b/src/archive/tar/format.go @@ -94,7 +94,7 @@ const ( // application can only parse GNU formatted archives. // // Reference: - // http://www.gnu.org/software/tar/manual/html_node/Standard.html + // https://www.gnu.org/software/tar/manual/html_node/Standard.html FormatGNU // Schily's tar format, which is incompatible with USTAR. diff --git a/src/archive/zip/struct.go b/src/archive/zip/struct.go index 36b551ec2c32a..c545c5b8308e9 100644 --- a/src/archive/zip/struct.go +++ b/src/archive/zip/struct.go @@ -202,7 +202,7 @@ func timeZone(offset time.Duration) *time.Location { // msDosTimeToTime converts an MS-DOS date and time into a time.Time. // The resolution is 2s. -// See: http://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx +// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx func msDosTimeToTime(dosDate, dosTime uint16) time.Time { return time.Date( // date bits 0-4: day of month; 5-8: month; 9-15: years since 1980 @@ -222,7 +222,7 @@ func msDosTimeToTime(dosDate, dosTime uint16) time.Time { // timeToMsDosTime converts a time.Time to an MS-DOS date and time. // The resolution is 2s. -// See: http://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx +// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) { fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9) fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11) diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 62ef872ca0da8..e9b79865650f1 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -725,7 +725,7 @@ func (p *Package) writeGccgoOutputFunc(fgcc *os.File, n *Name) { // packedAttribute returns host compiler struct attribute that will be // used to match gc's struct layout. For example, on 386 Windows, // gcc wants to 8-align int64s, but gc does not. -// Use __gcc_struct__ to work around http://gcc.gnu.org/PR52991 on x86, +// Use __gcc_struct__ to work around https://gcc.gnu.org/PR52991 on x86, // and https://golang.org/issue/5603. func (p *Package) packedAttribute() string { s := "__attribute__((__packed__" diff --git a/src/cmd/compile/internal/gc/phi.go b/src/cmd/compile/internal/gc/phi.go index bd66568eed025..5218cd0ef3d14 100644 --- a/src/cmd/compile/internal/gc/phi.go +++ b/src/cmd/compile/internal/gc/phi.go @@ -14,7 +14,7 @@ import ( // This file contains the algorithm to place phi nodes in a function. // For small functions, we use Braun, Buchwald, Hack, Leißa, Mallon, and Zwinkau. -// http://pp.info.uni-karlsruhe.de/uploads/publikationen/braun13cc.pdf +// https://pp.info.uni-karlsruhe.de/uploads/publikationen/braun13cc.pdf // For large functions, we use Sreedhar & Gao: A Linear Time Algorithm for Placing Φ-Nodes. // http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.8.1979&rep=rep1&type=pdf diff --git a/src/cmd/compile/internal/ssa/sparsemap.go b/src/cmd/compile/internal/ssa/sparsemap.go index c42fb99c7af8c..f55db54b1c0bf 100644 --- a/src/cmd/compile/internal/ssa/sparsemap.go +++ b/src/cmd/compile/internal/ssa/sparsemap.go @@ -6,7 +6,7 @@ package ssa import "cmd/internal/src" -// from http://research.swtch.com/sparse +// from https://research.swtch.com/sparse // in turn, from Briggs and Torczon type sparseEntry struct { diff --git a/src/cmd/compile/internal/ssa/sparseset.go b/src/cmd/compile/internal/ssa/sparseset.go index b5cabfb0cdf08..395931d1ff148 100644 --- a/src/cmd/compile/internal/ssa/sparseset.go +++ b/src/cmd/compile/internal/ssa/sparseset.go @@ -4,7 +4,7 @@ package ssa -// from http://research.swtch.com/sparse +// from https://research.swtch.com/sparse // in turn, from Briggs and Torczon type sparseSet struct { diff --git a/src/cmd/dist/sys_windows.go b/src/cmd/dist/sys_windows.go index 6d1f82e0936b6..216dc017982fe 100644 --- a/src/cmd/dist/sys_windows.go +++ b/src/cmd/dist/sys_windows.go @@ -14,7 +14,7 @@ var ( procGetSystemInfo = modkernel32.NewProc("GetSystemInfo") ) -// see http://msdn.microsoft.com/en-us/library/windows/desktop/ms724958(v=vs.85).aspx +// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms724958(v=vs.85).aspx type systeminfo struct { wProcessorArchitecture uint16 wReserved uint16 diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index 0f86834079d15..9b9df3cbc3ea0 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -2925,7 +2925,7 @@ func TestCgoPkgConfig(t *testing.T) { // OpenBSD's pkg-config is strict about whitespace and only // supports backslash-escaped whitespace. It does not support // quotes, which the normal freedesktop.org pkg-config does - // support. See http://man.openbsd.org/pkg-config.1 + // support. See https://man.openbsd.org/pkg-config.1 tg.tempFile("foo.pc", ` Name: foo Description: The foo library diff --git a/src/cmd/internal/obj/s390x/a.out.go b/src/cmd/internal/obj/s390x/a.out.go index ec959c40909fe..babcd2af010b2 100644 --- a/src/cmd/internal/obj/s390x/a.out.go +++ b/src/cmd/internal/obj/s390x/a.out.go @@ -150,7 +150,7 @@ const ( ) // LINUX for zSeries ELF Application Binary Interface Supplement -// http://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x1472.html +// https://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x1472.html var S390XDWARFRegisters = map[int16]int16{} func init() { diff --git a/src/cmd/link/internal/amd64/asm.go b/src/cmd/link/internal/amd64/asm.go index 6897ae21fe57f..af274444f36ae 100644 --- a/src/cmd/link/internal/amd64/asm.go +++ b/src/cmd/link/internal/amd64/asm.go @@ -617,7 +617,7 @@ func addpltsym(ctxt *ld.Link, s *sym.Symbol) { // so for now we'll just use non-lazy pointers, // which don't need to be told which library to use. // - // http://networkpx.blogspot.com/2009/09/about-lcdyldinfoonly-command.html + // https://networkpx.blogspot.com/2009/09/about-lcdyldinfoonly-command.html // has details about what we're avoiding. addgotsym(ctxt, s) diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index edf3922980698..816c867fa8f8a 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -1962,7 +1962,7 @@ func usage() { type SymbolType int8 const ( - // see also http://9p.io/magic/man2html/1/nm + // see also https://9p.io/magic/man2html/1/nm TextSym SymbolType = 'T' DataSym SymbolType = 'D' BSSSym SymbolType = 'B' diff --git a/src/cmd/link/internal/ld/macho.go b/src/cmd/link/internal/ld/macho.go index c0083fb8b0d3e..d804dc83b3428 100644 --- a/src/cmd/link/internal/ld/macho.go +++ b/src/cmd/link/internal/ld/macho.go @@ -143,7 +143,7 @@ const ( ) // Mach-O file writing -// http://developer.apple.com/mac/library/DOCUMENTATION/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html +// https://developer.apple.com/mac/library/DOCUMENTATION/DeveloperTools/Conceptual/MachORuntime/Reference/reference.html var machohdr MachoHdr diff --git a/src/cmd/link/internal/ld/pe.go b/src/cmd/link/internal/ld/pe.go index 66b1463086d73..8005dc5228412 100644 --- a/src/cmd/link/internal/ld/pe.go +++ b/src/cmd/link/internal/ld/pe.go @@ -116,7 +116,7 @@ const ( // license that can be found in the LICENSE file. // PE (Portable Executable) file writing -// http://www.microsoft.com/whdc/system/platform/firmware/PECOFF.mspx +// https://www.microsoft.com/whdc/system/platform/firmware/PECOFF.mspx // DOS stub that prints out // "This program cannot be run in DOS mode." diff --git a/src/cmd/link/internal/loadelf/ldelf.go b/src/cmd/link/internal/loadelf/ldelf.go index 7fb9a38a9fdfd..301c2ce1168e1 100644 --- a/src/cmd/link/internal/loadelf/ldelf.go +++ b/src/cmd/link/internal/loadelf/ldelf.go @@ -1048,7 +1048,7 @@ func readelfsym(arch *sys.Arch, syms *sym.Symbols, elfobj *ElfObj, i int, elfsym // __i686.get_pc_thunk.bx is allowed to be duplicated, to // workaround that we set dupok. // TODO(minux): correctly handle __i686.get_pc_thunk.bx without - // set dupok generally. See http://codereview.appspot.com/5823055/ + // set dupok generally. See https://golang.org/cl/5823055 // comment #5 for details. if s != nil && elfsym.other == 2 { s.Attr |= sym.AttrDuplicateOK | sym.AttrVisibilityHidden diff --git a/src/cmd/link/internal/wasm/asm.go b/src/cmd/link/internal/wasm/asm.go index 235a5a25d1453..aadb0c3b6ef1b 100644 --- a/src/cmd/link/internal/wasm/asm.go +++ b/src/cmd/link/internal/wasm/asm.go @@ -88,7 +88,7 @@ func assignAddress(ctxt *ld.Link, sect *sym.Section, n int, s *sym.Symbol, va ui } // asmb writes the final WebAssembly module binary. -// Spec: http://webassembly.github.io/spec/core/binary/modules.html +// Spec: https://webassembly.github.io/spec/core/binary/modules.html func asmb(ctxt *ld.Link) { if ctxt.Debugvlog != 0 { ctxt.Logf("%5.2f asmb\n", ld.Cputime()) diff --git a/src/compress/bzip2/bzip2.go b/src/compress/bzip2/bzip2.go index f07c7e81e876b..c40129b9820ee 100644 --- a/src/compress/bzip2/bzip2.go +++ b/src/compress/bzip2/bzip2.go @@ -8,7 +8,7 @@ package bzip2 import "io" // There's no RFC for bzip2. I used the Wikipedia page for reference and a lot -// of guessing: http://en.wikipedia.org/wiki/Bzip2 +// of guessing: https://en.wikipedia.org/wiki/Bzip2 // The source code to pyflate was useful for debugging: // http://www.paul.sladen.org/projects/pyflate diff --git a/src/compress/flate/deflate.go b/src/compress/flate/deflate.go index 4d6a5357d881d..8b92f1586db2c 100644 --- a/src/compress/flate/deflate.go +++ b/src/compress/flate/deflate.go @@ -720,7 +720,7 @@ func (w *Writer) Write(data []byte) (n int, err error) { // In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. func (w *Writer) Flush() error { // For more about flushing: - // http://www.bolet.org/~pornin/deflate-flush.html + // https://www.bolet.org/~pornin/deflate-flush.html return w.d.syncFlush() } diff --git a/src/compress/lzw/reader_test.go b/src/compress/lzw/reader_test.go index f8974de28fc95..98bbfbb763e4c 100644 --- a/src/compress/lzw/reader_test.go +++ b/src/compress/lzw/reader_test.go @@ -66,7 +66,7 @@ var lzwTests = []lzwTest{ "\x54\x9e\x08\x29\xf2\x44\x8a\x93\x27\x54\x04", io.ErrUnexpectedEOF, }, - // This example comes from http://en.wikipedia.org/wiki/Graphics_Interchange_Format. + // This example comes from https://en.wikipedia.org/wiki/Graphics_Interchange_Format. { "gif;LSB;8", "\x28\xff\xff\xff\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", diff --git a/src/compress/zlib/reader_test.go b/src/compress/zlib/reader_test.go index 7e27aecb47d1b..70e33babd1062 100644 --- a/src/compress/zlib/reader_test.go +++ b/src/compress/zlib/reader_test.go @@ -19,7 +19,7 @@ type zlibTest struct { } // Compare-to-golden test data was generated by the ZLIB example program at -// http://www.zlib.net/zpipe.c +// https://www.zlib.net/zpipe.c var zlibTests = []zlibTest{ { diff --git a/src/crypto/aes/aes_test.go b/src/crypto/aes/aes_test.go index 28144968fcfd2..bedc2da946593 100644 --- a/src/crypto/aes/aes_test.go +++ b/src/crypto/aes/aes_test.go @@ -122,7 +122,7 @@ func TestTd(t *testing.T) { } // Test vectors are from FIPS 197: -// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf +// https://csrc.nist.gov/publications/fips/fips197/fips-197.pdf // Appendix A of FIPS 197: Key expansion examples type KeyTest struct { diff --git a/src/crypto/aes/block.go b/src/crypto/aes/block.go index 41ea9cf95ed7c..8647019d5809a 100644 --- a/src/crypto/aes/block.go +++ b/src/crypto/aes/block.go @@ -31,8 +31,8 @@ // // See FIPS 197 for specification, and see Daemen and Rijmen's Rijndael submission // for implementation details. -// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf -// http://csrc.nist.gov/archive/aes/rijndael/Rijndael-ammended.pdf +// https://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf +// https://csrc.nist.gov/archive/aes/rijndael/Rijndael-ammended.pdf package aes diff --git a/src/crypto/aes/const.go b/src/crypto/aes/const.go index cbac5ff0ea155..4eca4b9aff842 100644 --- a/src/crypto/aes/const.go +++ b/src/crypto/aes/const.go @@ -15,7 +15,7 @@ package aes // This file contains AES constants - 8720 bytes of initialized data. -// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf +// https://csrc.nist.gov/publications/fips/fips197/fips-197.pdf // AES is based on the mathematical behavior of binary polynomials // (polynomials over GF(2)) modulo the irreducible polynomial x⁸ + x⁴ + x³ + x + 1. diff --git a/src/crypto/cipher/cfb_test.go b/src/crypto/cipher/cfb_test.go index 9b544bb2118b7..ecb716df01516 100644 --- a/src/crypto/cipher/cfb_test.go +++ b/src/crypto/cipher/cfb_test.go @@ -14,7 +14,7 @@ import ( ) // cfbTests contains the test vectors from -// http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf, section +// https://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf, section // F.3.13. var cfbTests = []struct { key, iv, plaintext, ciphertext string diff --git a/src/crypto/cipher/cipher.go b/src/crypto/cipher/cipher.go index 31c14d7f914a5..7e1a4de9a3778 100644 --- a/src/crypto/cipher/cipher.go +++ b/src/crypto/cipher/cipher.go @@ -4,7 +4,7 @@ // Package cipher implements standard block cipher modes that can be wrapped // around low-level block cipher implementations. -// See http://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html +// See https://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html // and NIST Special Publication 800-38A. package cipher diff --git a/src/crypto/cipher/gcm.go b/src/crypto/cipher/gcm.go index 0ea053428c009..c0ac9f163e83a 100644 --- a/src/crypto/cipher/gcm.go +++ b/src/crypto/cipher/gcm.go @@ -63,7 +63,7 @@ type gcmFieldElement struct { } // gcm represents a Galois Counter Mode with a specific key. See -// http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-revised-spec.pdf +// https://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-revised-spec.pdf type gcm struct { cipher Block nonceSize int diff --git a/src/crypto/ecdsa/ecdsa_test.go b/src/crypto/ecdsa/ecdsa_test.go index 9224a039f3feb..6284e06bd4390 100644 --- a/src/crypto/ecdsa/ecdsa_test.go +++ b/src/crypto/ecdsa/ecdsa_test.go @@ -213,7 +213,7 @@ func fromHex(s string) *big.Int { func TestVectors(t *testing.T) { // This test runs the full set of NIST test vectors from - // http://csrc.nist.gov/groups/STM/cavp/documents/dss/186-3ecdsatestvectors.zip + // https://csrc.nist.gov/groups/STM/cavp/documents/dss/186-3ecdsatestvectors.zip // // The SigVer.rsp file has been edited to remove test vectors for // unsupported algorithms and has been compressed. diff --git a/src/crypto/elliptic/elliptic.go b/src/crypto/elliptic/elliptic.go index 35aacf24e5152..4fc2b5e5213af 100644 --- a/src/crypto/elliptic/elliptic.go +++ b/src/crypto/elliptic/elliptic.go @@ -20,7 +20,7 @@ import ( ) // A Curve represents a short-form Weierstrass curve with a=-3. -// See http://www.hyperelliptic.org/EFD/g1p/auto-shortw.html +// See https://www.hyperelliptic.org/EFD/g1p/auto-shortw.html type Curve interface { // Params returns the parameters for the curve. Params() *CurveParams @@ -108,7 +108,7 @@ func (curve *CurveParams) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) { // addJacobian takes two points in Jacobian coordinates, (x1, y1, z1) and // (x2, y2, z2) and returns their sum, also in Jacobian form. func (curve *CurveParams) addJacobian(x1, y1, z1, x2, y2, z2 *big.Int) (*big.Int, *big.Int, *big.Int) { - // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl + // See https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl x3, y3, z3 := new(big.Int), new(big.Int), new(big.Int) if z1.Sign() == 0 { x3.Set(x2) @@ -191,7 +191,7 @@ func (curve *CurveParams) Double(x1, y1 *big.Int) (*big.Int, *big.Int) { // doubleJacobian takes a point in Jacobian coordinates, (x, y, z), and // returns its double, also in Jacobian form. func (curve *CurveParams) doubleJacobian(x, y, z *big.Int) (*big.Int, *big.Int, *big.Int) { - // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b + // See https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b delta := new(big.Int).Mul(z, z) delta.Mod(delta, curve.P) gamma := new(big.Int).Mul(y, y) diff --git a/src/crypto/elliptic/p224.go b/src/crypto/elliptic/p224.go index 22d0e2429cdfb..2ea63f3f0c057 100644 --- a/src/crypto/elliptic/p224.go +++ b/src/crypto/elliptic/p224.go @@ -7,7 +7,7 @@ package elliptic // This is a constant-time, 32-bit implementation of P224. See FIPS 186-3, // section D.2.2. // -// See http://www.imperialviolet.org/2010/12/04/ecc.html ([1]) for background. +// See https://www.imperialviolet.org/2010/12/04/ecc.html ([1]) for background. import ( "math/big" @@ -503,7 +503,7 @@ func p224Contract(out, in *p224FieldElement) { // p224AddJacobian computes *out = a+b where a != b. func p224AddJacobian(x3, y3, z3, x1, y1, z1, x2, y2, z2 *p224FieldElement) { - // See http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-p224Add-2007-bl + // See https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-p224Add-2007-bl var z1z1, z2z2, u1, u2, s1, s2, h, i, j, r, v p224FieldElement var c p224LargeFieldElement diff --git a/src/crypto/elliptic/p256.go b/src/crypto/elliptic/p256.go index bbf0087e664dd..bb9757355ac59 100644 --- a/src/crypto/elliptic/p256.go +++ b/src/crypto/elliptic/p256.go @@ -817,7 +817,7 @@ func p256Scalar8(out *[p256Limbs]uint32) { // p256PointDouble sets {xOut,yOut,zOut} = 2*{x,y,z}. // -// See http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-l +// See https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-l func p256PointDouble(xOut, yOut, zOut, x, y, z *[p256Limbs]uint32) { var delta, gamma, alpha, beta, tmp, tmp2 [p256Limbs]uint32 @@ -850,7 +850,7 @@ func p256PointDouble(xOut, yOut, zOut, x, y, z *[p256Limbs]uint32) { // p256PointAddMixed sets {xOut,yOut,zOut} = {x1,y1,z1} + {x2,y2,1}. // (i.e. the second point is affine.) // -// See http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl +// See https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl // // Note that this function does not handle P+P, infinity+P nor P+infinity // correctly. @@ -886,7 +886,7 @@ func p256PointAddMixed(xOut, yOut, zOut, x1, y1, z1, x2, y2 *[p256Limbs]uint32) // p256PointAdd sets {xOut,yOut,zOut} = {x1,y1,z1} + {x2,y2,z2}. // -// See http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl +// See https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl // // Note that this function does not handle P+P, infinity+P nor P+infinity // correctly. diff --git a/src/crypto/elliptic/p256_amd64.go b/src/crypto/elliptic/p256_amd64.go index b4346d7484c01..30eb33a0d4119 100644 --- a/src/crypto/elliptic/p256_amd64.go +++ b/src/crypto/elliptic/p256_amd64.go @@ -7,7 +7,7 @@ // detail in: // S.Gueron and V.Krasnov, "Fast prime field elliptic-curve cryptography with // 256-bit primes" -// http://link.springer.com/article/10.1007%2Fs13389-014-0090-x +// https://link.springer.com/article/10.1007%2Fs13389-014-0090-x // https://eprint.iacr.org/2013/816.pdf // +build amd64 diff --git a/src/crypto/elliptic/p256_asm_amd64.s b/src/crypto/elliptic/p256_asm_amd64.s index 4aebe37c8dd72..a4e375797718c 100644 --- a/src/crypto/elliptic/p256_asm_amd64.s +++ b/src/crypto/elliptic/p256_asm_amd64.s @@ -6,7 +6,7 @@ // P256. The optimizations performed here are described in detail in: // S.Gueron and V.Krasnov, "Fast prime field elliptic-curve cryptography with // 256-bit primes" -// http://link.springer.com/article/10.1007%2Fs13389-014-0090-x +// https://link.springer.com/article/10.1007%2Fs13389-014-0090-x // https://eprint.iacr.org/2013/816.pdf #include "textflag.h" diff --git a/src/crypto/elliptic/p256_asm_s390x.s b/src/crypto/elliptic/p256_asm_s390x.s index 8a17e81062fc5..2219b858b3b9b 100644 --- a/src/crypto/elliptic/p256_asm_s390x.s +++ b/src/crypto/elliptic/p256_asm_s390x.s @@ -1733,9 +1733,9 @@ TEXT ·p256PointAddAffineAsm(SB), NOSPLIT, $0 #undef CAR2 // p256PointDoubleAsm(P3, P1 *p256Point) -// http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian.html#doubling-dbl-2007-bl -// http://www.hyperelliptic.org/EFD/g1p/auto-shortw.html -// http://www.hyperelliptic.org/EFD/g1p/auto-shortw-projective-3.html +// https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian.html#doubling-dbl-2007-bl +// https://www.hyperelliptic.org/EFD/g1p/auto-shortw.html +// https://www.hyperelliptic.org/EFD/g1p/auto-shortw-projective-3.html #define P3ptr R1 #define P1ptr R2 #define CPOOL R4 @@ -1783,7 +1783,7 @@ TEXT ·p256PointAddAffineAsm(SB), NOSPLIT, $0 #define CAR1 V28 #define CAR2 V29 /* - * http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2004-hmv + * https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2004-hmv * Cost: 4M + 4S + 1*half + 5add + 2*2 + 1*3. * Source: 2004 Hankerson–Menezes–Vanstone, page 91. * A = 3(X₁-Z₁²)×(X₁+Z₁²) @@ -1995,7 +1995,7 @@ TEXT ·p256PointDoubleAsm(SB), NOSPLIT, $0 * Y₃ = D×(A×C² - X₃) - B×C³ * Z₃ = Z₁×Z₂×C * - * Three-operand formula (adopted): http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-1998-cmo-2 + * Three-operand formula (adopted): https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-1998-cmo-2 * Temp storage: T1,T2,U1,H,Z3=X3=Y3,S1,R * * T1 = Z1*Z1 diff --git a/src/crypto/hmac/hmac.go b/src/crypto/hmac/hmac.go index 3c8e727bc8c3d..c8c0617c47f78 100644 --- a/src/crypto/hmac/hmac.go +++ b/src/crypto/hmac/hmac.go @@ -27,7 +27,7 @@ import ( ) // FIPS 198-1: -// http://csrc.nist.gov/publications/fips/fips198-1/FIPS-198-1_final.pdf +// https://csrc.nist.gov/publications/fips/fips198-1/FIPS-198-1_final.pdf // key is zero padded to the block size of the hash function // ipad = 0x36 byte repeated for key length diff --git a/src/crypto/hmac/hmac_test.go b/src/crypto/hmac/hmac_test.go index aac9aa96a8ece..eea345edb635b 100644 --- a/src/crypto/hmac/hmac_test.go +++ b/src/crypto/hmac/hmac_test.go @@ -25,7 +25,7 @@ type hmacTest struct { var hmacTests = []hmacTest{ // Tests from US FIPS 198 - // http://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf + // https://csrc.nist.gov/publications/fips/fips198/fips-198a.pdf { sha1.New, []byte{ @@ -205,7 +205,7 @@ var hmacTests = []hmacTest{ sha256.BlockSize, }, - // Tests from http://csrc.nist.gov/groups/ST/toolkit/examples.html + // Tests from https://csrc.nist.gov/groups/ST/toolkit/examples.html // (truncated tag tests are left out) { sha1.New, diff --git a/src/crypto/rc4/rc4_test.go b/src/crypto/rc4/rc4_test.go index af7988246329d..1fc08b859343a 100644 --- a/src/crypto/rc4/rc4_test.go +++ b/src/crypto/rc4/rc4_test.go @@ -16,7 +16,7 @@ type rc4Test struct { var golden = []rc4Test{ // Test vectors from the original cypherpunk posting of ARC4: - // http://groups.google.com/group/sci.crypt/msg/10a300c9d21afca0?pli=1 + // https://groups.google.com/group/sci.crypt/msg/10a300c9d21afca0?pli=1 { []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, []byte{0x74, 0x94, 0xc2, 0xe7, 0x10, 0x4b, 0x08, 0x79}, @@ -30,7 +30,7 @@ var golden = []rc4Test{ []byte{0xd6, 0xa1, 0x41, 0xa7, 0xec, 0x3c, 0x38, 0xdf, 0xbd, 0x61}, }, - // Test vectors from the Wikipedia page: http://en.wikipedia.org/wiki/RC4 + // Test vectors from the Wikipedia page: https://en.wikipedia.org/wiki/RC4 { []byte{0x4b, 0x65, 0x79}, []byte{0xeb, 0x9f, 0x77, 0x81, 0xb7, 0x34, 0xca, 0x72, 0xa7, 0x19}, diff --git a/src/crypto/rsa/rsa.go b/src/crypto/rsa/rsa.go index 83d74967aa6dd..862657fa60958 100644 --- a/src/crypto/rsa/rsa.go +++ b/src/crypto/rsa/rsa.go @@ -68,7 +68,7 @@ var ( // We require pub.E to fit into a 32-bit integer so that we // do not have different behavior depending on whether // int is 32 or 64 bits. See also -// http://www.imperialviolet.org/2012/03/16/rsae.html. +// https://www.imperialviolet.org/2012/03/16/rsae.html. func checkPub(pub *PublicKey) error { if pub.N == nil { return errPublicModulus diff --git a/src/crypto/sha256/sha256block_386.s b/src/crypto/sha256/sha256block_386.s index 33ed027e1fcfc..086a0ab25c88c 100644 --- a/src/crypto/sha256/sha256block_386.s +++ b/src/crypto/sha256/sha256block_386.s @@ -6,7 +6,7 @@ // // The algorithm is detailed in FIPS 180-4: // -// http://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf +// https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf // // Wt = Mt; for 0 <= t <= 15 // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63 diff --git a/src/crypto/sha256/sha256block_amd64.s b/src/crypto/sha256/sha256block_amd64.s index f533f64260781..f6af47c50e9f0 100644 --- a/src/crypto/sha256/sha256block_amd64.s +++ b/src/crypto/sha256/sha256block_amd64.s @@ -8,7 +8,7 @@ // // The algorithm is detailed in FIPS 180-4: // -// http://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf +// https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf // The avx2-version is described in an Intel White-Paper: // "Fast SHA-256 Implementations on Intel Architecture Processors" diff --git a/src/crypto/sha256/sha256block_ppc64le.s b/src/crypto/sha256/sha256block_ppc64le.s index f5435602fe73b..77e63c073fd44 100644 --- a/src/crypto/sha256/sha256block_ppc64le.s +++ b/src/crypto/sha256/sha256block_ppc64le.s @@ -16,7 +16,7 @@ // // The algorithm is detailed in FIPS 180-4: // -// http://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf +// https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf // // Wt = Mt; for 0 <= t <= 15 // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63 diff --git a/src/crypto/sha512/sha512block_amd64.s b/src/crypto/sha512/sha512block_amd64.s index a02356607ec9a..0fa0df2f60e8b 100644 --- a/src/crypto/sha512/sha512block_amd64.s +++ b/src/crypto/sha512/sha512block_amd64.s @@ -8,7 +8,7 @@ // // The algorithm is detailed in FIPS 180-4: // -// http://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf +// https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf // // Wt = Mt; for 0 <= t <= 15 // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 79 @@ -274,7 +274,7 @@ end: // Version below is based on "Fast SHA512 Implementations on Intel // Architecture Processors" White-paper -// http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-sha512-implementations-ia-processors-paper.pdf +// https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-sha512-implementations-ia-processors-paper.pdf // AVX2 version by Intel, same algorithm in Linux kernel: // https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha512-avx2-asm.S diff --git a/src/crypto/sha512/sha512block_ppc64le.s b/src/crypto/sha512/sha512block_ppc64le.s index 170e3a645689e..55f0c06c7a056 100644 --- a/src/crypto/sha512/sha512block_ppc64le.s +++ b/src/crypto/sha512/sha512block_ppc64le.s @@ -16,7 +16,7 @@ // // The algorithm is detailed in FIPS 180-4: // -// http://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf +// https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf // // Wt = Mt; for 0 <= t <= 15 // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 79 diff --git a/src/crypto/tls/common.go b/src/crypto/tls/common.go index 32caa6233cbc0..76b1f6e5c4b37 100644 --- a/src/crypto/tls/common.go +++ b/src/crypto/tls/common.go @@ -246,19 +246,19 @@ type ClientHelloInfo struct { // ServerName indicates the name of the server requested by the client // in order to support virtual hosting. ServerName is only set if the // client is using SNI (see - // http://tools.ietf.org/html/rfc4366#section-3.1). + // https://tools.ietf.org/html/rfc4366#section-3.1). ServerName string // SupportedCurves lists the elliptic curves supported by the client. // SupportedCurves is set only if the Supported Elliptic Curves // Extension is being used (see - // http://tools.ietf.org/html/rfc4492#section-5.1.1). + // https://tools.ietf.org/html/rfc4492#section-5.1.1). SupportedCurves []CurveID // SupportedPoints lists the point formats supported by the client. // SupportedPoints is set only if the Supported Point Formats Extension // is being used (see - // http://tools.ietf.org/html/rfc4492#section-5.1.2). + // https://tools.ietf.org/html/rfc4492#section-5.1.2). SupportedPoints []uint8 // SignatureSchemes lists the signature and hash schemes that the client diff --git a/src/crypto/tls/conn.go b/src/crypto/tls/conn.go index dc3b8911c771d..cdaa7aba97c88 100644 --- a/src/crypto/tls/conn.go +++ b/src/crypto/tls/conn.go @@ -1061,9 +1061,9 @@ func (c *Conn) Write(b []byte) (int, error) { // This can be prevented by splitting each Application Data // record into two records, effectively randomizing the IV. // - // http://www.openssl.org/~bodo/tls-cbc.txt + // https://www.openssl.org/~bodo/tls-cbc.txt // https://bugzilla.mozilla.org/show_bug.cgi?id=665814 - // http://www.imperialviolet.org/2012/01/15/beastfollowup.html + // https://www.imperialviolet.org/2012/01/15/beastfollowup.html var m int if len(b) > 1 && c.vers <= VersionTLS10 { diff --git a/src/crypto/tls/handshake_messages.go b/src/crypto/tls/handshake_messages.go index f8c8d571ccde1..a5bf10efb8c6c 100644 --- a/src/crypto/tls/handshake_messages.go +++ b/src/crypto/tls/handshake_messages.go @@ -192,7 +192,7 @@ func (m *clientHelloMsg) marshal() []byte { z = z[9:] } if len(m.supportedCurves) > 0 { - // http://tools.ietf.org/html/rfc4492#section-5.5.1 + // https://tools.ietf.org/html/rfc4492#section-5.5.1 z[0] = byte(extensionSupportedCurves >> 8) z[1] = byte(extensionSupportedCurves) l := 2 + 2*len(m.supportedCurves) @@ -209,7 +209,7 @@ func (m *clientHelloMsg) marshal() []byte { } } if len(m.supportedPoints) > 0 { - // http://tools.ietf.org/html/rfc4492#section-5.5.2 + // https://tools.ietf.org/html/rfc4492#section-5.5.2 z[0] = byte(extensionSupportedPoints >> 8) z[1] = byte(extensionSupportedPoints) l := 1 + len(m.supportedPoints) @@ -224,7 +224,7 @@ func (m *clientHelloMsg) marshal() []byte { } } if m.ticketSupported { - // http://tools.ietf.org/html/rfc5077#section-3.2 + // https://tools.ietf.org/html/rfc5077#section-3.2 z[0] = byte(extensionSessionTicket >> 8) z[1] = byte(extensionSessionTicket) l := len(m.sessionTicket) @@ -414,7 +414,7 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { case extensionStatusRequest: m.ocspStapling = length > 0 && data[0] == statusTypeOCSP case extensionSupportedCurves: - // http://tools.ietf.org/html/rfc4492#section-5.5.1 + // https://tools.ietf.org/html/rfc4492#section-5.5.1 if length < 2 { return false } @@ -430,7 +430,7 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { d = d[2:] } case extensionSupportedPoints: - // http://tools.ietf.org/html/rfc4492#section-5.5.2 + // https://tools.ietf.org/html/rfc4492#section-5.5.2 if length < 1 { return false } @@ -441,7 +441,7 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { m.supportedPoints = make([]uint8, l) copy(m.supportedPoints, data[1:]) case extensionSessionTicket: - // http://tools.ietf.org/html/rfc5077#section-3.2 + // https://tools.ietf.org/html/rfc5077#section-3.2 m.ticketSupported = true m.sessionTicket = data[:length] case extensionSignatureAlgorithms: @@ -1224,7 +1224,7 @@ func (m *certificateRequestMsg) marshal() (x []byte) { return m.raw } - // See http://tools.ietf.org/html/rfc4346#section-7.4.4 + // See https://tools.ietf.org/html/rfc4346#section-7.4.4 length := 1 + len(m.certificateTypes) + 2 casLength := 0 for _, ca := range m.certificateAuthorities { @@ -1374,7 +1374,7 @@ func (m *certificateVerifyMsg) marshal() (x []byte) { return m.raw } - // See http://tools.ietf.org/html/rfc4346#section-7.4.8 + // See https://tools.ietf.org/html/rfc4346#section-7.4.8 siglength := len(m.signature) length := 2 + siglength if m.hasSignatureAndHash { @@ -1452,7 +1452,7 @@ func (m *newSessionTicketMsg) marshal() (x []byte) { return m.raw } - // See http://tools.ietf.org/html/rfc5077#section-3.3 + // See https://tools.ietf.org/html/rfc5077#section-3.3 ticketLen := len(m.ticket) length := 2 + 4 + ticketLen x = make([]byte, 4+length) diff --git a/src/crypto/tls/key_agreement.go b/src/crypto/tls/key_agreement.go index 3f570b66c6927..6685b47584f37 100644 --- a/src/crypto/tls/key_agreement.go +++ b/src/crypto/tls/key_agreement.go @@ -141,7 +141,7 @@ func pickTLS12HashForSignature(sigType uint8, clientList []SignatureScheme) (Sig if len(clientList) == 0 { // If the client didn't specify any signature_algorithms // extension then we can assume that it supports SHA1. See - // http://tools.ietf.org/html/rfc5246#section-7.4.1.4.1 + // https://tools.ietf.org/html/rfc5246#section-7.4.1.4.1 switch sigType { case signatureRSA: return PKCS1WithSHA1, nil @@ -239,7 +239,7 @@ NextCandidate: ecdhePublic = elliptic.Marshal(curve, x, y) } - // http://tools.ietf.org/html/rfc4492#section-5.4 + // https://tools.ietf.org/html/rfc4492#section-5.4 serverECDHParams := make([]byte, 1+2+1+len(ecdhePublic)) serverECDHParams[0] = 3 // named curve serverECDHParams[1] = byte(ka.curveid >> 8) diff --git a/src/crypto/tls/prf.go b/src/crypto/tls/prf.go index 93a638819dc34..367e0842b01c3 100644 --- a/src/crypto/tls/prf.go +++ b/src/crypto/tls/prf.go @@ -140,7 +140,7 @@ func prfForVersion(version uint16, suite *cipherSuite) func(result, secret, labe } // masterFromPreMasterSecret generates the master secret from the pre-master -// secret. See http://tools.ietf.org/html/rfc5246#section-8.1 +// secret. See https://tools.ietf.org/html/rfc5246#section-8.1 func masterFromPreMasterSecret(version uint16, suite *cipherSuite, preMasterSecret, clientRandom, serverRandom []byte) []byte { seed := make([]byte, 0, len(clientRandom)+len(serverRandom)) seed = append(seed, clientRandom...) diff --git a/src/crypto/x509/pkix/pkix.go b/src/crypto/x509/pkix/pkix.go index 7b32220b74e17..3cc4d587e363c 100644 --- a/src/crypto/x509/pkix/pkix.go +++ b/src/crypto/x509/pkix/pkix.go @@ -95,7 +95,7 @@ func (r RDNSequence) String() string { type RelativeDistinguishedNameSET []AttributeTypeAndValue // AttributeTypeAndValue mirrors the ASN.1 structure of the same name in -// http://tools.ietf.org/html/rfc5280#section-4.1.2.4 +// https://tools.ietf.org/html/rfc5280#section-4.1.2.4 type AttributeTypeAndValue struct { Type asn1.ObjectIdentifier Value interface{} diff --git a/src/crypto/x509/sha2_windows_test.go b/src/crypto/x509/sha2_windows_test.go index 79dc685c5b55b..620b7b9e77cb6 100644 --- a/src/crypto/x509/sha2_windows_test.go +++ b/src/crypto/x509/sha2_windows_test.go @@ -13,7 +13,7 @@ func init() { } if major := byte(v); major < 6 { // Windows XP SP2 and Windows 2003 do not support SHA2. - // http://blogs.technet.com/b/pki/archive/2010/09/30/sha2-and-windows.aspx + // https://blogs.technet.com/b/pki/archive/2010/09/30/sha2-and-windows.aspx supportSHA2 = false } } diff --git a/src/encoding/json/decode.go b/src/encoding/json/decode.go index 6a66940034645..0b29249218a32 100644 --- a/src/encoding/json/decode.go +++ b/src/encoding/json/decode.go @@ -204,7 +204,7 @@ func (n Number) Int64() (int64, error) { func isValidNumber(s string) bool { // This function implements the JSON numbers grammar. // See https://tools.ietf.org/html/rfc7159#section-6 - // and http://json.org/number.gif + // and https://json.org/number.gif if s == "" { return false diff --git a/src/encoding/json/number_test.go b/src/encoding/json/number_test.go index 4b8699963886e..cc6701814fb72 100644 --- a/src/encoding/json/number_test.go +++ b/src/encoding/json/number_test.go @@ -10,7 +10,7 @@ import ( ) func TestNumberIsValid(t *testing.T) { - // From: http://stackoverflow.com/a/13340826 + // From: https://stackoverflow.com/a/13340826 var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) validTests := []string{ diff --git a/src/encoding/xml/xml.go b/src/encoding/xml/xml.go index 7d2ff01ee946a..452caefab4da4 100644 --- a/src/encoding/xml/xml.go +++ b/src/encoding/xml/xml.go @@ -7,8 +7,8 @@ package xml // References: -// Annotated XML spec: http://www.xml.com/axml/testaxml.htm -// XML name spaces: http://www.w3.org/TR/REC-xml-names/ +// Annotated XML spec: https://www.xml.com/axml/testaxml.htm +// XML name spaces: https://www.w3.org/TR/REC-xml-names/ // TODO(rsc): // Test error handling. @@ -271,7 +271,7 @@ func NewTokenDecoder(t TokenReader) *Decoder { // it will return an error. // // Token implements XML name spaces as described by -// http://www.w3.org/TR/REC-xml-names/. Each of the +// https://www.w3.org/TR/REC-xml-names/. Each of the // Name structures contained in the Token has the Space // set to the URL identifying its name space when known. // If Token encounters an unrecognized name space prefix, @@ -863,7 +863,7 @@ func (d *Decoder) attrval() []byte { if !ok { return nil } - // http://www.w3.org/TR/REC-html40/intro/sgmltut.html#h-3.2.2 + // https://www.w3.org/TR/REC-html40/intro/sgmltut.html#h-3.2.2 if 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z' || '0' <= b && b <= '9' || b == '_' || b == ':' || b == '-' { d.buf.WriteByte(b) @@ -1134,7 +1134,7 @@ Input: } // Decide whether the given rune is in the XML Character Range, per -// the Char production of http://www.xml.com/axml/testaxml.htm, +// the Char production of https://www.xml.com/axml/testaxml.htm, // Section 2.2 Characters. func isInCharacterRange(r rune) (inrange bool) { return r == 0x09 || @@ -1263,7 +1263,7 @@ func isNameString(s string) bool { } // These tables were generated by cut and paste from Appendix B of -// the XML spec at http://www.xml.com/axml/testaxml.htm +// the XML spec at https://www.xml.com/axml/testaxml.htm // and then reformatting. First corresponds to (Letter | '_' | ':') // and second corresponds to NameChar. diff --git a/src/hash/crc32/crc32.go b/src/hash/crc32/crc32.go index 1912caa212ba1..908b84adcb290 100644 --- a/src/hash/crc32/crc32.go +++ b/src/hash/crc32/crc32.go @@ -3,12 +3,12 @@ // license that can be found in the LICENSE file. // Package crc32 implements the 32-bit cyclic redundancy check, or CRC-32, -// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for +// checksum. See https://en.wikipedia.org/wiki/Cyclic_redundancy_check for // information. // // Polynomials are represented in LSB-first form also known as reversed representation. // -// See http://en.wikipedia.org/wiki/Mathematics_of_cyclic_redundancy_checks#Reversed_representations_and_reciprocal_polynomials +// See https://en.wikipedia.org/wiki/Mathematics_of_cyclic_redundancy_checks#Reversed_representations_and_reciprocal_polynomials // for information. package crc32 @@ -29,12 +29,12 @@ const ( // Castagnoli's polynomial, used in iSCSI. // Has better error detection characteristics than IEEE. - // http://dx.doi.org/10.1109/26.231911 + // https://dx.doi.org/10.1109/26.231911 Castagnoli = 0x82f63b78 // Koopman's polynomial. // Also has better error detection characteristics than IEEE. - // http://dx.doi.org/10.1109/DSN.2002.1028931 + // https://dx.doi.org/10.1109/DSN.2002.1028931 Koopman = 0xeb31d82e ) diff --git a/src/hash/crc32/crc32_amd64.s b/src/hash/crc32/crc32_amd64.s index a944ead9b2f2e..6af6c253a7900 100644 --- a/src/hash/crc32/crc32_amd64.s +++ b/src/hash/crc32/crc32_amd64.s @@ -149,7 +149,7 @@ GLOBL r4r3<>(SB),RODATA,$16 GLOBL rupoly<>(SB),RODATA,$16 GLOBL r5<>(SB),RODATA,$8 -// Based on http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf +// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf // len(p) must be at least 64, and must be a multiple of 16. // func ieeeCLMUL(crc uint32, p []byte) uint32 diff --git a/src/hash/crc64/crc64.go b/src/hash/crc64/crc64.go index 3b24c2440628a..a799a017c938c 100644 --- a/src/hash/crc64/crc64.go +++ b/src/hash/crc64/crc64.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // Package crc64 implements the 64-bit cyclic redundancy check, or CRC-64, -// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for +// checksum. See https://en.wikipedia.org/wiki/Cyclic_redundancy_check for // information. package crc64 diff --git a/src/html/template/attr.go b/src/html/template/attr.go index 92d2789e8041e..22922e6038b65 100644 --- a/src/html/template/attr.go +++ b/src/html/template/attr.go @@ -13,9 +13,9 @@ import ( // other content, or affects the contents, idempotency, or credentials of a // network message, then the value in this map is contentTypeUnsafe. // This map is derived from HTML5, specifically -// http://www.w3.org/TR/html5/Overview.html#attributes-1 +// https://www.w3.org/TR/html5/Overview.html#attributes-1 // as well as "%URI"-typed attributes from -// http://www.w3.org/TR/html4/index/attributes.html +// https://www.w3.org/TR/html4/index/attributes.html var attrTypeMap = map[string]contentType{ "accept": contentTypePlain, "accept-charset": contentTypeUnsafe, @@ -90,7 +90,7 @@ var attrTypeMap = map[string]contentType{ "name": contentTypePlain, "novalidate": contentTypeUnsafe, // Skip handler names from - // http://www.w3.org/TR/html5/webappapis.html#event-handlers-on-elements,-document-objects,-and-window-objects + // https://www.w3.org/TR/html5/webappapis.html#event-handlers-on-elements,-document-objects,-and-window-objects // since we have special handling in attrType. "open": contentTypePlain, "optimum": contentTypePlain, @@ -160,7 +160,7 @@ func attrType(name string) contentType { // Heuristics to prevent "javascript:..." injection in custom // data attributes and custom attributes like g:tweetUrl. - // http://www.w3.org/TR/html5/dom.html#embedding-custom-non-visible-data-with-the-data-*-attributes + // https://www.w3.org/TR/html5/dom.html#embedding-custom-non-visible-data-with-the-data-*-attributes // "Custom data attributes are intended to store custom data // private to the page or application, for which there are no // more appropriate attributes or elements." diff --git a/src/html/template/content.go b/src/html/template/content.go index e7cdedc3b6209..4aadf64df252f 100644 --- a/src/html/template/content.go +++ b/src/html/template/content.go @@ -16,7 +16,7 @@ type ( // 2. The CSS3 rule production, such as `a[href=~"https:"].foo#bar`. // 3. CSS3 declaration productions, such as `color: red; margin: 2px`. // 4. The CSS3 value production, such as `rgba(0, 0, 255, 127)`. - // See http://www.w3.org/TR/css3-syntax/#parsing and + // See https://www.w3.org/TR/css3-syntax/#parsing and // https://web.archive.org/web/20090211114933/http://w3.org/TR/css3-syntax#style // // Use of this type presents a security risk: @@ -85,7 +85,7 @@ type ( URL string // Srcset encapsulates a known safe srcset attribute - // (see http://w3c.github.io/html/semantics-embedded-content.html#element-attrdef-img-srcset). + // (see https://w3c.github.io/html/semantics-embedded-content.html#element-attrdef-img-srcset). // // Use of this type presents a security risk: // the encapsulated content should come from a trusted source, diff --git a/src/html/template/context.go b/src/html/template/context.go index fdbf7e25ee2df..45be3a6a9f94d 100644 --- a/src/html/template/context.go +++ b/src/html/template/context.go @@ -13,7 +13,7 @@ import ( // // The zero value of type context is the start context for a template that // produces an HTML fragment as defined at -// http://www.w3.org/TR/html5/syntax.html#the-end +// https://www.w3.org/TR/html5/syntax.html#the-end // where the context element is null. type context struct { state state @@ -98,7 +98,7 @@ const ( // stateHTMLCmt occurs inside an . stateHTMLCmt // stateRCDATA occurs inside an RCDATA element (