Basic HTTP client.

R=rsc
APPROVED=rsc
DELTA=392  (386 added, 2 deleted, 4 changed)
OCL=29963
CL=30107

src/pkg/http/Makefile

@@ -39,6 +39,7 @@ O2=\
 	request.$O\
 
 O3=\
+	client.$O\
 	server.$O\
 
 O4=\
@@ -57,7 +58,7 @@ a2: $(O2)
 	rm -f $(O2)
 
 a3: $(O3)
-	$(AR) grc _obj$D/http.a server.$O
+	$(AR) grc _obj$D/http.a client.$O server.$O
 	rm -f $(O3)
 
 a4: $(O4)

src/pkg/http/client.go

@@ -0,0 +1,227 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Primitive HTTP client.  See RFC 2616.
+
+package http
+
+import (
+	"bufio";
+	"fmt";
+	"http";
+	"io";
+	"log";
+	"net";
+	"os";
+	"strings";
+	"strconv";
+)
+
+// Response represents the response from an HTTP request.
+type Response struct {
+	Status string; // e.g. "200 OK"
+	StatusCode int; // e.g. 200
+
+	// Header maps header keys to values.  If the response had multiple
+	// headers with the same key, they will be concatenated, with comma
+	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
+	// be semantically equivalent to a comma-delimited sequence.)
+	//
+	// Keys in the map are canonicalized (see CanonicalHeaderKey).
+	Header map [string] string;
+
+	// Stream from which the response body can be read.
+	Body io.ReadCloser;
+}
+
+// GetHeader returns the value of the response header with the given
+// key, and true.  If there were multiple headers with this key, their
+// values are concatenated, with a comma delimiter.  If there were no
+// response headers with the given key, it returns the empty string and
+// false.  Keys are not case sensitive.
+func (r *Response) GetHeader(key string) (value string) {
+	value, present := r.Header[CanonicalHeaderKey(key)];
+	return;
+}
+
+// AddHeader adds a value under the given key.  Keys are not case sensitive.
+func (r *Response) AddHeader(key, value string) {
+	key = CanonicalHeaderKey(key);
+
+	oldValues, oldValuesPresent := r.Header[key];
+	if oldValuesPresent {
+		r.Header[key] = oldValues + "," + value;
+	} else {
+		r.Header[key] = value;
+	}
+}
+
+// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
+// return true if the string includes a port.
+func hasPort(s string) bool {
+	return strings.LastIndex(s, ":") > strings.LastIndex(s, "]");
+}
+
+// Used in Send to implement io.ReadCloser by bundling together the
+// io.BufReader through which we read the response, and the underlying
+// network connection.
+type readClose struct {
+	io.Reader;
+	io.Closer;
+}
+
+// Send issues an HTTP request.  Caller should close resp.Body when done reading it.
+//
+// This method consults the following fields of req:
+//
+//	Url
+//	Method (defaults to "GET")
+//	Proto (defaults to "HTTP/1.0")
+//	UserAgent (if empty, currently defaults to http.Client; may change)
+//	Referer (if empty, no Referer header will be supplied)
+//	Header
+//	Body (if nil, defaults to empty body)
+//
+// The following fields are redundant and are ignored:
+//
+//	RawUrl
+//	ProtoMajor
+//	ProtoMinor
+//	Close
+//	Host
+//
+// TODO: support persistent connections (multiple requests on a single connection).
+// send() method is nonpublic because, when we refactor the code for persistent
+// connections, it may no longer make sense to have a method with this signature.
+func send(req *Request) (resp *Response, err os.Error) {
+	if req.Url.Scheme != "http" {
+		return nil, os.ErrorString("Unsupported protocol: " + req.Url.Scheme);
+	}
+
+	addr := req.Url.Host;
+	if !hasPort(addr) {
+		addr += ":http";
+	}
+	conn, err := net.Dial("tcp", "", addr);
+	if err != nil {
+		return nil, os.ErrorString("Error dialing " + addr + ": " + err.String());
+	}
+
+	// Close the connection if we encounter an error during header parsing.  We'll
+	// cancel this when we hand the connection off to our caller.
+	defer func() { if conn != nil { conn.Close() } }();
+
+	err = req.write(conn);
+	if err != nil {
+		return nil, err;
+	}
+
+	// Parse the first line of the response.
+	resp = new(Response);
+	resp.Header = make(map[string] string);
+	reader := bufio.NewReader(conn);
+
+	line, err := readLine(reader);
+	if err != nil {
+		return nil, err;
+	}
+	ss := strings.Split(line, " ");
+	if len(ss) != 3 {
+		return nil, os.ErrorString(fmt.Sprintf("Invalid first line in HTTP response: %q", line));
+	}
+	resp.Status = ss[1] + " " + ss[2];
+	resp.StatusCode, err = strconv.Atoi(ss[1]);
+	if err != nil {
+		return nil, os.ErrorString(fmt.Sprintf("Invalid status code in HTTP response %q", line));
+	}
+
+	// Parse the response headers.
+	for {
+		key, value, err := readKeyValue(reader);
+		if err != nil {
+			return nil, err;
+		}
+		if key == "" {
+			break; // end of response header
+		}
+		resp.AddHeader(key, value);
+	}
+
+	resp.Body = readClose{reader, conn};
+	conn = nil; // so that defered func won't close it
+	err = nil;
+	return;
+}
+
+// True if the specified HTTP status code is one for which the Get utility should
+// automatically redirect.
+func shouldRedirect(statusCode int) bool {
+	switch statusCode {
+	case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect:
+		return true;
+	}
+	return false;
+}
+
+// Get issues a GET to the specified URL.  If the response is one of the following
+// redirect codes, it follows the redirect, up to a maximum of 10 redirects:
+//
+//    301 (Moved Permanently)
+//    302 (Found)
+//    303 (See Other)
+//    307 (Temporary Redirect)
+//
+// finalUrl is the URL from which the response was fetched -- identical to the input
+// URL unless redirects were followed.
+//
+// Caller should close r.Body when done reading it.
+func Get(url string) (r *Response, finalUrl string, err os.Error) {
+	// TODO: if/when we add cookie support, the redirected request shouldn't
+	// necessarily supply the same cookies as the original.
+	// TODO: adjust referrer header on redirects.
+	for redirectCount := 0; redirectCount < 10; redirectCount++ {
+		var req Request;
+		req.Url, err = ParseURL(url);
+		if err != nil {
+			return nil, url, err;
+		}
+
+		r, err := send(&req);
+		if err != nil {
+			return nil, url, err;
+		}
+
+		if !shouldRedirect(r.StatusCode) {
+			return r, url, nil;
+		}
+
+		r.Body.Close();
+		url := r.GetHeader("Location");
+		if url == "" {
+			return r, url, os.ErrorString("302 result with no Location header");
+		}
+	}
+
+	return nil, url, os.ErrorString("Too many redirects");
+}
+
+
+// Post issues a POST to the specified URL.
+//
+// Caller should close resp.Body when done reading it.
+func Post(url string, requestBody io.Reader) (r *Response, err os.Error) {
+	// NOTE TO REVIEWER: this could share more code with Get, waiting for API to settle
+	// down before cleaning up that detail.
+
+	var req Request;
+	req.Method = "POST";
+	req.Body = requestBody;
+	req.Url, err = ParseURL(url);
+	if err != nil {
+		return nil, err;
+	}
+
+	return send(&req);
+}
+

src/pkg/http/client_test.go

@@ -0,0 +1,40 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Tests for client.go
+
+package http
+
+import (
+	"fmt";
+	"http";
+	"io";
+	"strings";
+	"testing";
+)
+
+func TestClient(t *testing.T) {
+	// TODO: add a proper test suite.  Current test merely verifies that
+	// we can retrieve the Google home page.
+
+	r, url, err := Get("http://www.google.com");
+	var b []byte;
+	if err == nil {
+		b, err = io.ReadAll(r.Body);
+		r.Body.Close();
+	}
+
+	// TODO: io.ErrEOF check is needed because we're sometimes getting
+	// this error when nothing is actually wrong.  rsc suspects a bug
+	// in bufio.  Can remove the ErrEOF check once the bug is fixed
+	// (expected to occur within a few weeks of this writing, 6/9/09).
+	if err != nil && err != io.ErrEOF {
+		t.Errorf("Error fetching URL: %v", err);
+	} else {
+		s := string(b);
+		if (!strings.HasPrefix(s, "<html>")) {
+			t.Errorf("Incorrect page body (did not begin with <html>): %q", s);
+		}
+	}
+}

src/pkg/http/request.go

@@ -4,11 +4,9 @@
 
 // HTTP Request reading and parsing.
 
-// The http package implements parsing of HTTP requests and URLs
-// and provides an extensible HTTP server.
-//
-// In the future it should also implement parsing of HTTP replies
-// and provide methods to fetch URLs via HTTP.
+// The http package implements parsing of HTTP requests, replies,
+// and URLs and provides an extensible HTTP server and a basic
+// HTTP client.
 package http
 
 import (
@@ -106,6 +104,56 @@ func (r *Request) ProtoAtLeast(major, minor int) bool {
 		r.ProtoMajor == major && r.ProtoMinor >= minor
 }
 
+// Return value if nonempty, def otherwise.
+func valueOrDefault(value, def string) string {
+	if value != "" {
+		return value;
+	}
+	return def;
+}
+
+// TODO(rsc): Change default UserAgent before open-source release.
+const defaultUserAgent = "http.Client";
+
+// Write an HTTP request -- header and body -- in wire format.
+// See Send for a list of which Request fields we use.
+func (req *Request) write(w io.Writer) os.Error {
+	uri := "/" + URLEscape(req.Url.Path);
+	if req.Url.RawQuery != "" {
+		uri += "?" + req.Url.RawQuery;
+	}
+
+	fmt.Fprintf(w, "%s %s %s\r\n", valueOrDefault(req.Method, "GET"), uri, valueOrDefault(req.Proto, "HTTP/1.0"));
+	fmt.Fprintf(w, "Host: %s\r\n", req.Url.Host);
+	fmt.Fprintf(w, "User-Agent: %s\r\n", valueOrDefault(req.UserAgent, defaultUserAgent));
+
+	if (req.Referer != "") {
+		fmt.Fprintf(w, "Referer: %s\r\n", req.Referer);
+	}
+
+	// TODO: split long values?  (If so, should share code with Conn.Write)
+	// TODO: if Header includes values for Host, User-Agent, or Referer, this
+	// may conflict with the User-Agent or Referer headers we add manually.
+	// One solution would be to remove the Host, UserAgent, and Referer fields
+	// from Request, and introduce Request methods along the lines of
+	// Response.{GetHeader,AddHeader} and string constants for "Host",
+	// "User-Agent" and "Referer".
+	for k, v := range req.Header {
+		io.WriteString(w, k + ": " + v + "\r\n");
+	}
+
+	io.WriteString(w, "\r\n");
+
+	if req.Body != nil {
+		nCopied, err := io.Copy(req.Body, w);
+		if err != nil && err != io.ErrEOF {
+			return err;
+		}
+	}
+
+	return nil;
+}
+
 // Read a line of bytes (up to \n) from b.
 // Give up if the line exceeds maxLineLength.
 // The returned bytes are a pointer into storage in

src/pkg/strings/strings.go

@@ -53,6 +53,21 @@ func Index(s, sep string) int {
 	return -1
 }
 
+// Index returns the index of the last instance of sep in s, or -1 if sep is not present in s.
+func LastIndex(s, sep string) int {
+	n := len(sep);
+	if n == 0 {
+		return len(s)
+	}
+	c := sep[0];
+	for i := len(s)-n; i >= 0; i-- {
+		if s[i] == c && (n == 1 || s[i:i+n] == sep) {
+			return i
+		}
+	}
+	return -1
+}
+
 // Split returns the array representing the substrings of s separated by string sep. Adjacent
 // occurrences of sep produce empty substrings.  If sep is empty, it is the same as Explode.
 func Split(s, sep string) []string {