[fix] update request data after redirect - closes #74

colly.go

@@ -322,6 +322,10 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c
 	if err := c.handleOnError(response, err, request, ctx); err != nil {
 		return err
 	}
+	if req.URL.String() != parsedURL.String() {
+		request.URL = req.URL
+		request.Headers = &req.Header
+	}
 	atomic.AddUint32(&c.responseCount, 1)
 	response.Ctx = ctx
 	response.Request = request

colly_test.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/PuerkitoBio/goquery"
@@ -70,6 +71,14 @@ func init() {
 		w.Write([]byte("disallowed"))
 	})
 
+	http.Handle("/redirect", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/redirected/", http.StatusSeeOther)
+
+	}))
+	http.Handle("/redirected/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, `<a href="test">test</a>`)
+	}))
+
 	http.HandleFunc("/set_cookie", func(w http.ResponseWriter, r *http.Request) {
 		c := &http.Cookie{Name: "test", Value: "testv", HttpOnly: false}
 		http.SetCookie(w, c)
@@ -228,6 +237,17 @@ func TestCollectorPost(t *testing.T) {
 	})
 }
 
+func TestRedirect(t *testing.T) {
+	c := NewCollector()
+	c.OnHTML("a[href]", func(e *HTMLElement) {
+		u := e.Request.AbsoluteURL(e.Attr("href"))
+		if !strings.HasSuffix(u, "/redirected/test") {
+			t.Error("Invalid URL after redirect: " + u)
+		}
+	})
+	c.Visit(testServerRootURL + "redirect")
+}
+
 func TestCollectorCookies(t *testing.T) {
 	c := NewCollector()
 

http_backend.go

@@ -165,6 +165,7 @@ func (h *httpBackend) Do(request *http.Request, bodySize int) (*Response, error)
 	if err != nil {
 		return nil, err
 	}
+	*request = *res.Request
 
 	var bodyReader io.Reader = res.Body
 	if bodySize > 0 {