From 241e317e089a10c567dfdfaa12e9d544efb3fcf7 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Thu, 18 May 2023 13:41:53 -0700 Subject: [PATCH 1/6] add ErrClosed error type for natiu-mqtt --- net.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/net.go b/net.go index cf88c6b..bd6cff4 100644 --- a/net.go +++ b/net.go @@ -197,3 +197,27 @@ func (e *AddrError) Error() string { } return s } + +// errNetClosing is the type of the variable ErrNetClosing. +// This is used to implement the net.Error interface. +type errNetClosing struct{} + +// Error returns the error message for ErrNetClosing. +// Keep this string consistent because of issue #4373: +// since historically programs have not been able to detect +// this error, they look for the string. +func (e errNetClosing) Error() string { return "use of closed network connection" } + +func (e errNetClosing) Timeout() bool { return false } +func (e errNetClosing) Temporary() bool { return false } + +// errClosed exists just so that the docs for ErrClosed don't mention +// the internal package poll. +var errClosed = errNetClosing{} + +// ErrClosed is the error returned by an I/O call on a network +// connection that has already been closed, or that is closed by +// another goroutine before the I/O is completed. This may be wrapped +// in another error, and should normally be tested using +// errors.Is(err, net.ErrClosed). +var ErrClosed error = errClosed From 488c3560165f4ebce44606142ad26dba7ebc44e3 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Thu, 8 Jun 2023 23:56:41 -0700 Subject: [PATCH 2/6] netdev: move GetIPAddr() from netlink to netdev Netlink is for L2; netdev is for L3/L4 --- netdev.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/netdev.go b/netdev.go index 95d42db..c3922bc 100644 --- a/netdev.go +++ b/netdev.go @@ -49,6 +49,10 @@ type netdever interface { // address in standard dot notation GetHostByName(name string) (IP, error) + // GetIPAddr returns IP address assigned to the interface, either by + // DHCP or statically + GetIPAddr() (IP, error) + // Berkely Sockets-like interface, Go-ified. See man page for socket(2), etc. Socket(domain int, stype int, protocol int) (int, error) Bind(sockfd int, ip IP, port int) error From a9f90b2846535e2a4f8efc8ef793a22eb5cdd9ec Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Thu, 8 Jun 2023 23:59:53 -0700 Subject: [PATCH 3/6] netdev: improve documentation Call out netdev as L3/L4 interface for TinyGo, to differentiate between netlink which is stricly L2. --- netdev.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/netdev.go b/netdev.go index c3922bc..c503e98 100644 --- a/netdev.go +++ b/netdev.go @@ -1,3 +1,5 @@ +// L3/L4 network/transport layer + package net import ( @@ -28,20 +30,19 @@ func useNetdev(dev netdever) { netdev = dev } -// Netdev is TinyGo's network device driver model. Network drivers implement -// the netdever interface, providing a common network I/O interface to TinyGo's -// "net" package. The interface is modeled after the BSD socket interface. -// net.Conn implementations (TCPConn, UDPConn, and TLSConn) use the netdev -// interface for device I/O access. +// netdever is TinyGo's OSI L3/L4 network/transport layer interface. Network +// drivers implement the netdever interface, providing a common network L3/L4 +// interface to TinyGo's "net" package. net.Conn implementations (TCPConn, +// UDPConn, and TLSConn) use the netdever interface for device I/O access. // // A netdever is passed to the "net" package using net.useNetdev(). // // Just like a net.Conn, multiple goroutines may invoke methods on a netdever // simultaneously. // -// NOTE: The netdever interface is mirrored in drivers/netdev.go. +// NOTE: The netdever interface is mirrored in drivers/netdev/netdev.go. // NOTE: If making changes to this interface, mirror the changes in -// NOTE: drivers/netdev.go, and vice-versa. +// NOTE: drivers/netdev/netdev.go, and vice-versa. type netdever interface { From 3e1636f756fa84a872f6dfb5dd435b7b0e18d3cf Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Mon, 26 Jun 2023 20:34:40 -0700 Subject: [PATCH 4/6] README: add documentation on maintaining 'net' package --- README.md | 66 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index bba8fdb..d84565b 100644 --- a/README.md +++ b/README.md @@ -3,17 +3,22 @@ package. The subset maintains Go 1 compatiblity guarantee. The "net" package is modified to use netdev, TinyGo's network device driver interface. Netdev replaces the OS syscall interface for I/O access to the networking -device. +device. See drivers/netdev for more information on netdev. #### Table of Contents -- ["net" Package](#net-package) -- [Netdev and Netlink](#netdev-and-netlink) - [Using "net" and "net/http" Packages](#using-net-and-nethttp-packages) +- ["net" Package](#net-package) +- [Maintaining "net"](#maintaining-net) +## Using "net" and "net/http" Packages + +See README-net.md in drivers repo to more details on using "net" and "net/http" +packages in a TinyGo application. + ## "net" Package -The "net" package is ported from Go 1.19.3. The tree listings below shows the +The "net" package is ported from Go 1.20.5. The tree listings below shows the files copied. If the file is marked with an '\*', it is copied _and_ modified to work with netdev. If the file is marked with an '+', the file is new. If there is no mark, it is a straight copy. @@ -74,7 +79,7 @@ request/response handling code is intact and operational in TinyGo. Same holds true for the server side. The server side supports the normal server features like ServeMux and Hijacker (for websockets). -### Maintaining "net" +## Maintaining "net" As Go progresses, changes to the "net" package need to be periodically back-ported to TinyGo's "net" package. This is to pick up any upstream bug @@ -86,22 +91,45 @@ The files that are marked modified * may contain only a subset of the original file. Basically only the parts necessary to compile and run the example/net examples are copied (and maybe modified). -## Netdev and Netlink +### Upgrade Steps -Netdev is TinyGo's network device driver model. Network drivers implement the -netdever interface, providing a common network I/O interface to TinyGo's "net" -package. The interface is modeled after the BSD socket interface. net.Conn -implementations (TCPConn, UDPConn, and TLSConn) use the netdev interface for -device I/O access. +Let's define some versions: -Network drivers also (optionally) implement the Netlinker interface. This -interface is not used by TinyGo's "net" package, but rather provides the TinyGo -application direct access to the network device for common settings and control -that fall outside of netdev's socket interface. +MIN = TinyGo minimum Go version supported (e.g. 1.15) +CUR = TinyGo "net" current version (e.g. 1.20.5) +UPSTREAM = Latest upstream Go version to upgrade to (e.g. 1.21) +NEW = TinyGo "net" new version, after upgrade -See the README-net.md in drivers repo for more details on netdev and netlink. +In example, we'll upgrade from CUR (1.20.5) to UPSTREAM (1.21). -## Using "net" and "net/http" Packages +These are the steps to promote TinyGos "net" to latest Go upstream version. +These steps should be done when: -See README-net.md in drivers repo to more details on using "net" and "net/http" -packages in a TinyGo application. +- MIN moved forward +- TinyGo major release +- TinyGo minor release to pick up security fixes in UPSTREAM + +Step 1: + +Backport differences from Go UPSTREAM to Go CUR. Since TinyGo CUR isn't the +full Go "net" implementation, only backport differences, don't add new stuff +from UPSTREAM (unless it's needed in the NEW release). + + NEW = CUR + diff(CUR, UPSTREAM) + +If NEW contains updates not compatible with MIN, then NEW will need to revert +just those updates back to the CUR version, and annotate with a TINYGO comment. +If MIN moves forord, NEW can pull in the UPSTREAM changes. + +Step 2: + +As a double check, compare NEW against UPSTREAM. The only differences at this +point should be excluded (not ported) code from UPSTREAM that wasn't in CUR in +the first place, and differences due to changes held back for MIN support. + +Step 3: + +Test NEW against example/net examples. If everything checks out, then CUR +becomes NEW, and we can push to TinyGo. + + CUR = NEW From fb7bbf483e0060d78586f83f52528492c888b478 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Mon, 26 Jun 2023 20:36:40 -0700 Subject: [PATCH 5/6] update 'net' package from 1.19.3 to 1.20.5 --- dial.go | 2 +- http/client.go | 2 +- http/clone.go | 2 +- http/cookie.go | 2 +- http/fs.go | 162 ++++++++++++++++-------------- http/header.go | 2 +- http/http.go | 2 +- http/internal/ascii/print.go | 2 +- http/internal/ascii/print_test.go | 2 + http/internal/chunked.go | 2 +- http/internal/chunked_test.go | 2 + http/jar.go | 2 +- http/method.go | 2 +- http/request.go | 28 ++++-- http/response.go | 2 +- http/server.go | 102 +++++++++---------- http/sniff.go | 2 +- http/status.go | 2 +- http/transfer.go | 15 +-- http/transport.go | 2 +- ip.go | 2 +- iprawsock.go | 2 +- ipsock.go | 2 +- mac.go | 2 +- mac_test.go | 2 + net.go | 2 +- parse.go | 30 ++---- pipe.go | 2 +- tcpsock.go | 2 +- tlssock.go | 2 +- udpsock.go | 2 +- 31 files changed, 195 insertions(+), 194 deletions(-) diff --git a/dial.go b/dial.go index ac32e62..940552c 100644 --- a/dial.go +++ b/dial.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // TINYGO: Omit DualStack support // TINYGO: Omit Fast Fallback support diff --git a/http/client.go b/http/client.go index 8bfef71..2863c83 100644 --- a/http/client.go +++ b/http/client.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/clone.go b/http/clone.go index aa42a7e..4980b8d 100644 --- a/http/clone.go +++ b/http/clone.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/cookie.go b/http/cookie.go index 24c938c..70e1eb3 100644 --- a/http/cookie.go +++ b/http/cookie.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/fs.go b/http/fs.go index 3967045..6796917 100644 --- a/http/fs.go +++ b/http/fs.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -11,6 +11,7 @@ package http import ( "errors" "fmt" + "internal/safefilepath" "io" "io/fs" "mime" @@ -71,14 +72,15 @@ func mapOpenError(originalErr error, name string, sep rune, stat func(string) (f // Open implements FileSystem using os.Open, opening files for reading rooted // and relative to the directory d. func (d Dir) Open(name string) (File, error) { - if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { - return nil, errors.New("http: invalid character in file path") + path, err := safefilepath.FromFS(path.Clean("/" + name)) + if err != nil { + return nil, errors.New("http: invalid or unsafe file path") } dir := string(d) if dir == "" { dir = "." } - fullName := filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))) + fullName := filepath.Join(dir, path) f, err := os.Open(fullName) if err != nil { return nil, mapOpenError(err, fullName, filepath.Separator, os.Stat) @@ -256,81 +258,95 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, Error(w, err.Error(), StatusInternalServerError) return } + if size < 0 { + // Should never happen but just to be sure + Error(w, "negative content size computed", StatusInternalServerError) + return + } // handle Content-Range header. sendSize := size var sendContent io.Reader = content - if size >= 0 { - ranges, err := parseRange(rangeReq, size) - if err != nil { - if err == errNoOverlap { - w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) - } + ranges, err := parseRange(rangeReq, size) + switch err { + case nil: + case errNoOverlap: + if size == 0 { + // Some clients add a Range header to all requests to + // limit the size of the response. If the file is empty, + // ignore the range header and respond with a 200 rather + // than a 416. + ranges = nil + break + } + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) + fallthrough + default: + Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) + return + } + + if sumRangesSize(ranges) > size { + // The total number of bytes in all the ranges + // is larger than the size of the file by + // itself, so this is probably an attack, or a + // dumb client. Ignore the range request. + ranges = nil + } + switch { + case len(ranges) == 1: + // RFC 7233, Section 4.1: + // "If a single part is being transferred, the server + // generating the 206 response MUST generate a + // Content-Range header field, describing what range + // of the selected representation is enclosed, and a + // payload consisting of the range. + // ... + // A server MUST NOT generate a multipart response to + // a request for a single range, since a client that + // does not request multiple parts might not support + // multipart responses." + ra := ranges[0] + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) return } - if sumRangesSize(ranges) > size { - // The total number of bytes in all the ranges - // is larger than the size of the file by - // itself, so this is probably an attack, or a - // dumb client. Ignore the range request. - ranges = nil - } - switch { - case len(ranges) == 1: - // RFC 7233, Section 4.1: - // "If a single part is being transferred, the server - // generating the 206 response MUST generate a - // Content-Range header field, describing what range - // of the selected representation is enclosed, and a - // payload consisting of the range. - // ... - // A server MUST NOT generate a multipart response to - // a request for a single range, since a client that - // does not request multiple parts might not support - // multipart responses." - ra := ranges[0] - if _, err := content.Seek(ra.start, io.SeekStart); err != nil { - Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) - return - } - sendSize = ra.length - code = StatusPartialContent - w.Header().Set("Content-Range", ra.contentRange(size)) - case len(ranges) > 1: - sendSize = rangesMIMESize(ranges, ctype, size) - code = StatusPartialContent - - pr, pw := io.Pipe() - mw := multipart.NewWriter(pw) - w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) - sendContent = pr - defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. - go func() { - for _, ra := range ranges { - part, err := mw.CreatePart(ra.mimeHeader(ctype, size)) - if err != nil { - pw.CloseWithError(err) - return - } - if _, err := content.Seek(ra.start, io.SeekStart); err != nil { - pw.CloseWithError(err) - return - } - if _, err := io.CopyN(part, content, ra.length); err != nil { - pw.CloseWithError(err) - return - } + sendSize = ra.length + code = StatusPartialContent + w.Header().Set("Content-Range", ra.contentRange(size)) + case len(ranges) > 1: + sendSize = rangesMIMESize(ranges, ctype, size) + code = StatusPartialContent + + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) + sendContent = pr + defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. + go func() { + for _, ra := range ranges { + part, err := mw.CreatePart(ra.mimeHeader(ctype, size)) + if err != nil { + pw.CloseWithError(err) + return } - mw.Close() - pw.Close() - }() - } + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.CopyN(part, content, ra.length); err != nil { + pw.CloseWithError(err) + return + } + } + mw.Close() + pw.Close() + }() + } - w.Header().Set("Accept-Ranges", "bytes") - if w.Header().Get("Content-Encoding") == "" { - w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) - } + w.Header().Set("Accept-Ranges", "bytes") + if w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) } w.WriteHeader(code) @@ -433,7 +449,7 @@ func checkIfUnmodifiedSince(r *Request, modtime time.Time) condResult { // The Last-Modified header truncates sub-second precision so // the modtime needs to be truncated too. modtime = modtime.Truncate(time.Second) - if modtime.Before(t) || modtime.Equal(t) { + if ret := modtime.Compare(t); ret <= 0 { return condTrue } return condFalse @@ -484,7 +500,7 @@ func checkIfModifiedSince(r *Request, modtime time.Time) condResult { // The Last-Modified header truncates sub-second precision so // the modtime needs to be truncated too. modtime = modtime.Truncate(time.Second) - if modtime.Before(t) || modtime.Equal(t) { + if ret := modtime.Compare(t); ret <= 0 { return condFalse } return condTrue @@ -644,7 +660,6 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec defer ff.Close() dd, err := ff.Stat() if err == nil { - name = index d = dd f = ff } @@ -820,6 +835,7 @@ func (f ioFile) Readdir(count int) ([]fs.FileInfo, error) { // FS converts fsys to a FileSystem implementation, // for use with FileServer and NewFileTransport. +// The files provided by fsys must implement io.Seeker. func FS(fsys fs.FS) FileSystem { return ioFS{fsys} } diff --git a/http/header.go b/http/header.go index a5779f6..ac3d0bb 100644 --- a/http/header.go +++ b/http/header.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // TINYGO: Removed trace stuff diff --git a/http/http.go b/http/http.go index fc1db57..900d5a9 100644 --- a/http/http.go +++ b/http/http.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/internal/ascii/print.go b/http/internal/ascii/print.go index c2b3a9b..b71ed91 100644 --- a/http/internal/ascii/print.go +++ b/http/internal/ascii/print.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2021 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/internal/ascii/print_test.go b/http/internal/ascii/print_test.go index 0b7767c..fb84a2e 100644 --- a/http/internal/ascii/print_test.go +++ b/http/internal/ascii/print_test.go @@ -1,3 +1,5 @@ +// TINYGO: The following is copied from Go 1.20.5 official implementation. + // Copyright 2021 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/http/internal/chunked.go b/http/internal/chunked.go index 34b5331..f363820 100644 --- a/http/internal/chunked.go +++ b/http/internal/chunked.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/internal/chunked_test.go b/http/internal/chunked_test.go index 5e29a78..002e74d 100644 --- a/http/internal/chunked_test.go +++ b/http/internal/chunked_test.go @@ -1,3 +1,5 @@ +// TINYGO: The following is copied from Go 1.20.5 official implementation. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/http/jar.go b/http/jar.go index 3091c58..6be849e 100644 --- a/http/jar.go +++ b/http/jar.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/method.go b/http/method.go index b8a4c33..5b09e98 100644 --- a/http/method.go +++ b/http/method.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/request.go b/http/request.go index 1971ec4..897918a 100644 --- a/http/request.go +++ b/http/request.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // TINYGO: Removed multipart stuff // TINYGO: Removed trace stuff @@ -51,9 +51,12 @@ type ProtocolError struct { func (pe *ProtocolError) Error() string { return pe.ErrorString } var ( - // ErrNotSupported is returned by the Push method of Pusher - // implementations to indicate that HTTP/2 Push support is not - // available. + // ErrNotSupported indicates that a feature is not supported. + // + // It is returned by ResponseController methods to indicate that + // the handler does not support the method, and by the Push method + // of Pusher implementations to indicate that HTTP/2 Push support + // is not available. ErrNotSupported = &ProtocolError{"feature not supported"} // Deprecated: ErrUnexpectedTrailer is no longer returned by @@ -319,7 +322,7 @@ type Request struct { Response *Response // ctx is either the client or server context. It should only - // be modified via copying the whole Request using WithContext. + // be modified via copying the whole Request using Clone or WithContext. // It is unexported to prevent people from using Context wrong // and mutating the contexts held by callers of the same request. ctx context.Context @@ -330,7 +333,7 @@ type Request struct { } // Context returns the request's context. To change the context, use -// WithContext. +// Clone or WithContext. // // The returned context is always non-nil; it defaults to the // background context. @@ -355,9 +358,7 @@ func (r *Request) Context() context.Context { // sending the request, and reading the response headers and body. // // To create a new request with a context, use NewRequestWithContext. -// To change the context of a request, such as an incoming request you -// want to modify before sending back out, use Request.Clone. Between -// those two uses, it's rare to need WithContext. +// To make a deep copy of a request with a new context, use Request.Clone. func (r *Request) WithContext(ctx context.Context) *Request { if ctx == nil { panic("nil context") @@ -424,6 +425,9 @@ var ErrNoCookie = errors.New("http: named cookie not present") // If multiple cookies match the given name, only one cookie will // be returned. func (r *Request) Cookie(name string) (*Cookie, error) { + if name == "" { + return nil, ErrNoCookie + } for _, c := range readCookies(r.Header, name) { return c, nil } @@ -993,6 +997,8 @@ func ReadRequest(b *bufio.Reader) (*Request, error) { func readRequest(b *bufio.Reader) (req *Request, err error) { tp := newTextprotoReader(b) + defer putTextprotoReader(tp) + req = new(Request) // First line: GET /index.html HTTP/1.0 @@ -1001,7 +1007,6 @@ func readRequest(b *bufio.Reader) (req *Request, err error) { return nil, err } defer func() { - putTextprotoReader(tp) if err == io.EOF { err = io.ErrUnexpectedEOF } @@ -1132,7 +1137,8 @@ func (l *maxBytesReader) Read(p []byte) (n int, err error) { // If they asked for a 32KB byte read but only 5 bytes are // remaining, no need to read 32KB. 6 bytes will answer the // question of the whether we hit the limit or go past it. - if int64(len(p)) > l.n+1 { + // 0 < len(p) < 2^63 + if int64(len(p))-1 > l.n { p = p[:l.n+1] } n, err = l.r.Read(p) diff --git a/http/response.go b/http/response.go index 980329f..30ffe7e 100644 --- a/http/response.go +++ b/http/response.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // TINYGO: Removed TLS connection state // TINYGO: Added onEOF hook to get callback when response has been read diff --git a/http/server.go b/http/server.go index 1a4264d..c12d197 100644 --- a/http/server.go +++ b/http/server.go @@ -1,4 +1,8 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. + +// TINYGO: atomic.Pointer and atomic.Uint64 are added in Go 1.19, so keep +// pre-1.19 code to cover min TinyGo. If TinyGo min moves to 1.19 or higher, +// then these can be converted to atomic.Pointer and atomic.Uint64. // TINYGO: Removed ALPN protocol support // TINYGO: Removed some HTTP/2 support @@ -400,11 +404,11 @@ func (cw *chunkWriter) Write(p []byte) (n int, err error) { return } -func (cw *chunkWriter) flush() { +func (cw *chunkWriter) flush() error { if !cw.wroteHeader { cw.writeHeader(nil) } - cw.res.conn.bufw.Flush() + return cw.res.conn.bufw.Flush() } func (cw *chunkWriter) close() { @@ -435,9 +439,9 @@ type response struct { wants10KeepAlive bool // HTTP/1.0 w/ Connection "keep-alive" wantsClose bool // HTTP request has Connection "close" - // canWriteContinue is a boolean value accessed as an atomic int32 - // that says whether or not a 100 Continue header can be written - // to the connection. + // canWriteContinue is an atomic boolean that says whether or + // not a 100 Continue header can be written to the + // connection. // writeContinueMu must be held while writing the header. // These two fields together synchronize the body reader (the // expectContinueReader, which wants to write 100 Continue) @@ -494,6 +498,14 @@ type response struct { didCloseNotify int32 // atomic (only 0->1 winner should send) } +func (c *response) SetReadDeadline(deadline time.Time) error { + return c.conn.rwc.SetReadDeadline(deadline) +} + +func (c *response) SetWriteDeadline(deadline time.Time) error { + return c.conn.rwc.SetWriteDeadline(deadline) +} + // TrailerPrefix is a magic prefix for ResponseWriter.Header map keys // that, if present, signals that the map entry is actually for // the response trailers, and not the response headers. The prefix @@ -514,11 +526,11 @@ const TrailerPrefix = "Trailer:" func (w *response) finalTrailers() Header { var t Header for k, vv := range w.handlerHeader { - if strings.HasPrefix(k, TrailerPrefix) { + if kk, found := strings.CutPrefix(k, TrailerPrefix); found { if t == nil { t = make(Header) } - t[strings.TrimPrefix(k, TrailerPrefix)] = vv + t[kk] = vv } } for _, k := range w.trailers { @@ -560,12 +572,6 @@ func (w *response) requestTooLarge() { } } -// needsSniff reports whether a Content-Type still needs to be sniffed. -func (w *response) needsSniff() bool { - _, haveType := w.handlerHeader["Content-Type"] - return !w.cw.wroteHeader && !haveType && w.written < sniffLen -} - // writerOnly hides an io.Writer value's optional ReadFrom method // from io.Copy. type writerOnly struct { @@ -754,8 +760,8 @@ func (cr *connReader) handleReadError(_ error) { // may be called from multiple goroutines. func (cr *connReader) closeNotify() { - res, _ := cr.conn.curReq.Load().(*response) - if res != nil && atomic.CompareAndSwapInt32(&res.didCloseNotify, 0, 1) { + res := cr.conn.curReq.Load() + if res != nil && !res.didCloseNotify.Swap(true) { res.closeNotifyCh <- true } } @@ -1706,11 +1712,19 @@ func (w *response) closedRequestBodyEarly() bool { } func (w *response) Flush() { + w.FlushError() +} + +func (w *response) FlushError() error { if !w.wroteHeader { w.WriteHeader(StatusOK) } - w.w.Flush() - w.cw.flush() + err := w.w.Flush() + e2 := w.cw.flush() + if err == nil { + err = e2 + } + return err } func (c *conn) finalFlush() { @@ -1964,6 +1978,7 @@ func (c *conn) serve(ctx context.Context) { return } w.finishRequest() + c.rwc.SetWriteDeadline(time.Time{}) if !w.shouldReuseConnection() { if w.requestBodyLimitHit || w.closedRequestBodyEarly() { c.closeWriteAndWait() @@ -1983,10 +1998,18 @@ func (c *conn) serve(ctx context.Context) { if d := c.server.idleTimeout(); d != 0 { c.rwc.SetReadDeadline(time.Now().Add(d)) - if _, err := c.bufr.Peek(4); err != nil { - return - } + } else { + c.rwc.SetReadDeadline(time.Time{}) } + + // Wait for the connection to become readable again before trying to + // read the next request. This prevents a ReadHeaderTimeout or + // ReadTimeout from starting until the first bytes of the next request + // have been received. + if _, err := c.bufr.Peek(4); err != nil { + return + } + c.rwc.SetReadDeadline(time.Time{}) } } @@ -2548,6 +2571,10 @@ type Server struct { Handler Handler // handler to invoke, http.DefaultServeMux if nil + // DisableGeneralOptionsHandler, if true, passes "OPTIONS *" requests to the Handler, + // otherwise responds with 200 OK and Content-Length: 0. + DisableGeneralOptionsHandler bool + // TLSConfig optionally provides a TLS configuration for use // by ServeTLS and ListenAndServeTLS. Note that this value is // cloned by ServeTLS and ListenAndServeTLS, so it's not @@ -2629,37 +2656,11 @@ type Server struct { mu sync.Mutex listeners map[*net.Listener]struct{} activeConn map[*conn]struct{} - doneChan chan struct{} onShutdown []func() listenerGroup sync.WaitGroup } -func (s *Server) getDoneChan() <-chan struct{} { - s.mu.Lock() - defer s.mu.Unlock() - return s.getDoneChanLocked() -} - -func (s *Server) getDoneChanLocked() chan struct{} { - if s.doneChan == nil { - s.doneChan = make(chan struct{}) - } - return s.doneChan -} - -func (s *Server) closeDoneChanLocked() { - ch := s.getDoneChanLocked() - select { - case <-ch: - // Already closed. Don't close again. - default: - // Safe to close here. We're the only closer, guarded - // by s.mu. - close(ch) - } -} - // Close immediately closes all active net.Listeners and any // connections in state StateNew, StateActive, or StateIdle. For a // graceful shutdown, use Shutdown. @@ -2725,7 +2726,6 @@ func (srv *Server) Shutdown(ctx context.Context) error { srv.mu.Lock() lnerr := srv.closeListenersLocked() - srv.closeDoneChanLocked() for _, f := range srv.onShutdown { go f() } @@ -2869,7 +2869,7 @@ func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) { if handler == nil { handler = DefaultServeMux } - if req.RequestURI == "*" && req.Method == "OPTIONS" { + if !sh.srv.DisableGeneralOptionsHandler && req.RequestURI == "*" && req.Method == "OPTIONS" { handler = globalOptionsHandler{} } @@ -2984,10 +2984,8 @@ func (srv *Server) Serve(l net.Listener) error { for { rw, err := l.Accept() if err != nil { - select { - case <-srv.getDoneChan(): + if srv.shuttingDown() { return ErrServerClosed - default: } if ne, ok := err.(net.Error); ok && ne.Temporary() { if tempDelay == 0 { diff --git a/http/sniff.go b/http/sniff.go index 3fee912..66c55b3 100644 --- a/http/sniff.go +++ b/http/sniff.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/status.go b/http/status.go index cd3db73..ec4f1ee 100644 --- a/http/status.go +++ b/http/status.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/http/transfer.go b/http/transfer.go index 38bd70f..e528968 100644 --- a/http/transfer.go +++ b/http/transfer.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // TINYGO: Removed trace stuff // TINYGO: Hook readTransfer is onEOF callback to get notified when request of @@ -553,7 +553,7 @@ func readTransfer(msg any, r *bufio.Reader, onEOF func()) (err error) { // or close connection when finished, since multipart is not supported yet switch { case t.Chunked: - if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { + if isResponse && (noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode)) { t.Body = NoBody } else { t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close, onHitEOF: onEOF} @@ -596,7 +596,7 @@ func readTransfer(msg any, r *bufio.Reader, onEOF func()) (err error) { return nil } -// Checks whether chunked is part of the encodings stack +// Checks whether chunked is part of the encodings stack. func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } // Checks whether the encoding is explicitly "identity". @@ -687,14 +687,7 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, } // Logic based on response type or status - if noResponseBodyExpected(requestMethod) { - // For HTTP requests, as part of hardening against request - // smuggling (RFC 7230), don't allow a Content-Length header for - // methods which don't permit bodies. As an exception, allow - // exactly one Content-Length header if its value is "0". - if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { - return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens) - } + if isResponse && noResponseBodyExpected(requestMethod) { return 0, nil } if status/100 == 1 { diff --git a/http/transport.go b/http/transport.go index a5f0f49..ae29865 100644 --- a/http/transport.go +++ b/http/transport.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/ip.go b/ip.go index 1a3a0b6..16477a6 100644 --- a/ip.go +++ b/ip.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/iprawsock.go b/iprawsock.go index 8f82ec8..f39850e 100644 --- a/iprawsock.go +++ b/iprawsock.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/ipsock.go b/ipsock.go index 52d1f7d..3ab9ebf 100644 --- a/ipsock.go +++ b/ipsock.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/mac.go b/mac.go index 320b209..2c855cb 100644 --- a/mac.go +++ b/mac.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/mac_test.go b/mac_test.go index cad884f..8bd8ba8 100644 --- a/mac_test.go +++ b/mac_test.go @@ -1,3 +1,5 @@ +// TINYGO: The following is copied from Go 1.20.5 official implementation. + // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/net.go b/net.go index bd6cff4..e66e19b 100644 --- a/net.go +++ b/net.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/parse.go b/parse.go index b263271..1a5e6c1 100644 --- a/parse.go +++ b/parse.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied from Go 1.19.3 official implementation. +// TINYGO: The following is copied from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -238,7 +238,7 @@ func lowerASCII(b byte) byte { } // trimSpace returns x without any leading or trailing ASCII whitespace. -func trimSpace(x []byte) []byte { +func trimSpace(x string) string { for len(x) > 0 && isSpace(x[0]) { x = x[1:] } @@ -255,37 +255,19 @@ func isSpace(b byte) bool { // removeComment returns line, removing any '#' byte and any following // bytes. -func removeComment(line []byte) []byte { - if i := bytealg.IndexByte(line, '#'); i != -1 { +func removeComment(line string) string { + if i := bytealg.IndexByteString(line, '#'); i != -1 { return line[:i] } return line } -// foreachLine runs fn on each line of x. -// Each line (except for possibly the last) ends in '\n'. -// It returns the first non-nil error returned by fn. -func foreachLine(x []byte, fn func(line []byte) error) error { - for len(x) > 0 { - nl := bytealg.IndexByte(x, '\n') - if nl == -1 { - return fn(x) - } - line := x[:nl+1] - x = x[nl+1:] - if err := fn(line); err != nil { - return err - } - } - return nil -} - // foreachField runs fn on each non-empty run of non-space bytes in x. // It returns the first non-nil error returned by fn. -func foreachField(x []byte, fn func(field []byte) error) error { +func foreachField(x string, fn func(field string) error) error { x = trimSpace(x) for len(x) > 0 { - sp := bytealg.IndexByte(x, ' ') + sp := bytealg.IndexByteString(x, ' ') if sp == -1 { return fn(x) } diff --git a/pipe.go b/pipe.go index 238da0c..b61a4dc 100644 --- a/pipe.go +++ b/pipe.go @@ -1,4 +1,4 @@ -// The following is copied from Go 1.19.3 official implementation. +// The following is copied from Go 1.20.5 official implementation. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/tcpsock.go b/tcpsock.go index 42af855..90605b7 100644 --- a/tcpsock.go +++ b/tcpsock.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/tlssock.go b/tlssock.go index f4f228b..639e998 100644 --- a/tlssock.go +++ b/tlssock.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/udpsock.go b/udpsock.go index f782c5f..086668b 100644 --- a/udpsock.go +++ b/udpsock.go @@ -1,4 +1,4 @@ -// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. +// TINYGO: The following is copied and modified from Go 1.20.5 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style From 16539031d660f4108de4849f3f96e2eb2d385bcc Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Tue, 27 Jun 2023 02:25:12 -0700 Subject: [PATCH 6/6] revert some changes going from 1.19.3 to 1.20.5 There are some new features added in Go 1.20 like string.CutPrefix. Those will have to wait to be ported until TinyGo MIN is >= 1.20. --- http/fs.go | 16 +++++++++------- http/server.go | 10 +++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/http/fs.go b/http/fs.go index 6796917..ed8250c 100644 --- a/http/fs.go +++ b/http/fs.go @@ -11,7 +11,6 @@ package http import ( "errors" "fmt" - "internal/safefilepath" "io" "io/fs" "mime" @@ -72,15 +71,16 @@ func mapOpenError(originalErr error, name string, sep rune, stat func(string) (f // Open implements FileSystem using os.Open, opening files for reading rooted // and relative to the directory d. func (d Dir) Open(name string) (File, error) { - path, err := safefilepath.FromFS(path.Clean("/" + name)) - if err != nil { - return nil, errors.New("http: invalid or unsafe file path") + // TINYGO: internal/safefilepath isn't avail until 1.20, so keep pre 1.20 + // TINYGO: code here until TinyGo min Go version is >= 1.20. + if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { + return nil, errors.New("http: invalid character in file path") } dir := string(d) if dir == "" { dir = "." } - fullName := filepath.Join(dir, path) + fullName := filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))) f, err := os.Open(fullName) if err != nil { return nil, mapOpenError(err, fullName, filepath.Separator, os.Stat) @@ -449,7 +449,8 @@ func checkIfUnmodifiedSince(r *Request, modtime time.Time) condResult { // The Last-Modified header truncates sub-second precision so // the modtime needs to be truncated too. modtime = modtime.Truncate(time.Second) - if ret := modtime.Compare(t); ret <= 0 { + // TINYGO: time.Compare not until Go 1.20 + if modtime.Before(t) || modtime.Equal(t) { return condTrue } return condFalse @@ -500,7 +501,8 @@ func checkIfModifiedSince(r *Request, modtime time.Time) condResult { // The Last-Modified header truncates sub-second precision so // the modtime needs to be truncated too. modtime = modtime.Truncate(time.Second) - if ret := modtime.Compare(t); ret <= 0 { + // TINYGO: time.Compare not until Go 1.20 + if modtime.Before(t) || modtime.Equal(t) { return condFalse } return condTrue diff --git a/http/server.go b/http/server.go index c12d197..3992765 100644 --- a/http/server.go +++ b/http/server.go @@ -526,11 +526,12 @@ const TrailerPrefix = "Trailer:" func (w *response) finalTrailers() Header { var t Header for k, vv := range w.handlerHeader { - if kk, found := strings.CutPrefix(k, TrailerPrefix); found { + // TINYGO: CutPrefix not available until 1.20 + if strings.HasPrefix(k, TrailerPrefix) { if t == nil { t = make(Header) } - t[kk] = vv + t[strings.TrimPrefix(k, TrailerPrefix)] = vv } } for _, k := range w.trailers { @@ -760,8 +761,8 @@ func (cr *connReader) handleReadError(_ error) { // may be called from multiple goroutines. func (cr *connReader) closeNotify() { - res := cr.conn.curReq.Load() - if res != nil && !res.didCloseNotify.Swap(true) { + res, _ := cr.conn.curReq.Load().(*response) + if res != nil && atomic.CompareAndSwapInt32(&res.didCloseNotify, 0, 1) { res.closeNotifyCh <- true } } @@ -2674,7 +2675,6 @@ func (srv *Server) Close() error { srv.inShutdown.setTrue() srv.mu.Lock() defer srv.mu.Unlock() - srv.closeDoneChanLocked() err := srv.closeListenersLocked() // Unlock srv.mu while waiting for listenerGroup.