From c35fa9a2778168fdb3b81f7bc089d35ed157eb01 Mon Sep 17 00:00:00 2001 From: Andy Goldstein Date: Tue, 21 Jul 2015 16:45:39 -0400 Subject: [PATCH 01/13] Add http proxy support for exec/port-forward Add http proxy support for exec/port-forward in SpdyRoundTripper --- pkg/util/httpstream/spdy/roundtripper.go | 71 ++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/pkg/util/httpstream/spdy/roundtripper.go b/pkg/util/httpstream/spdy/roundtripper.go index b00678221cf8..35147edb2f15 100644 --- a/pkg/util/httpstream/spdy/roundtripper.go +++ b/pkg/util/httpstream/spdy/roundtripper.go @@ -23,6 +23,8 @@ import ( "io/ioutil" "net" "net/http" + "net/http/httputil" + "net/url" "strings" "k8s.io/kubernetes/pkg/api" @@ -53,21 +55,82 @@ type SpdyRoundTripper struct { Dialer *net.Dialer } -// NewSpdyRoundTripper creates a new SpdyRoundTripper that will use +// NewRoundTripper creates a new SpdyRoundTripper that will use // the specified tlsConfig. func NewRoundTripper(tlsConfig *tls.Config) httpstream.UpgradeRoundTripper { return NewSpdyRoundTripper(tlsConfig) } +// NewSpdyRoundTripper creates a new SpdyRoundTripper that will use +// the specified tlsConfig. This function is mostly meant for unit tests. func NewSpdyRoundTripper(tlsConfig *tls.Config) *SpdyRoundTripper { return &SpdyRoundTripper{tlsConfig: tlsConfig} } -// dial dials the host specified by req, using TLS if appropriate. +// dial dials the host specified by req, using TLS if appropriate, optionally +// using a proxy server if one is configured via environment variables. func (s *SpdyRoundTripper) dial(req *http.Request) (net.Conn, error) { - dialAddr := netutil.CanonicalAddr(req.URL) + proxyURL, err := http.ProxyFromEnvironment(req) + if err != nil { + return nil, err + } + + if proxyURL == nil { + return s.dialWithoutProxy(req.URL) + } + + // proxying logic adapted from http://blog.h6t.eu/post/74098062923/golang-websocket-with-http-proxy-support + proxyReq := http.Request{ + Method: "CONNECT", + URL: &url.URL{}, + Host: req.URL.Host, + } + + proxyDialConn, err := s.dialWithoutProxy(proxyURL) + if err != nil { + return nil, err + } + + proxyClientConn := httputil.NewProxyClientConn(proxyDialConn, nil) + _, err = proxyClientConn.Do(&proxyReq) + if err != nil && err != httputil.ErrPersistEOF { + return nil, err + } + + rwc, _ := proxyClientConn.Hijack() + + if req.URL.Scheme != "https" { + return rwc, nil + } + + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + return nil, err + } + + if len(s.tlsConfig.ServerName) == 0 { + s.tlsConfig.ServerName = host + } + + tlsConn := tls.Client(rwc, s.tlsConfig) + + // need to manually call Handshake() so we can call VerifyHostname() below + if err := tlsConn.Handshake(); err != nil { + return nil, err + } + + if err := tlsConn.VerifyHostname(host); err != nil { + return nil, err + } + + return tlsConn, nil +} + +// dialWithoutProxy dials the host specified by url, using TLS if appropriate. +func (s *SpdyRoundTripper) dialWithoutProxy(url *url.URL) (net.Conn, error) { + dialAddr := netutil.CanonicalAddr(url) - if req.URL.Scheme == "http" { + if url.Scheme == "http" { if s.Dialer == nil { return net.Dial("tcp", dialAddr) } else { From 12d6be493eee73f8aa1a60af6b26caf6fbe6a3b1 Mon Sep 17 00:00:00 2001 From: Andy Goldstein Date: Thu, 27 Aug 2015 10:19:29 -0400 Subject: [PATCH 02/13] Add goproxy test image --- Godeps/Godeps.json | 5 + .../src/github.com/elazarl/goproxy/.gitignore | 2 + .../src/github.com/elazarl/goproxy/LICENSE | 27 + .../src/github.com/elazarl/goproxy/README.md | 115 ++ .../src/github.com/elazarl/goproxy/actions.go | 57 + .../src/github.com/elazarl/goproxy/all.bash | 15 + .../src/github.com/elazarl/goproxy/ca.pem | 15 + .../src/github.com/elazarl/goproxy/certs.go | 56 + .../src/github.com/elazarl/goproxy/chunked.go | 59 + .../elazarl/goproxy/counterecryptor.go | 68 + .../elazarl/goproxy/counterecryptor_test.go | 99 + .../src/github.com/elazarl/goproxy/ctx.go | 87 + .../github.com/elazarl/goproxy/dispatcher.go | 320 ++++ .../src/github.com/elazarl/goproxy/doc.go | 100 + .../goproxy/examples/goproxy-basic/README.md | 29 + .../goproxy/examples/goproxy-basic/main.go | 17 + .../examples/goproxy-eavesdropper/main.go | 56 + .../examples/goproxy-httpdump/README.md | 30 + .../examples/goproxy-httpdump/httpdump.go | 285 +++ .../examples/goproxy-jquery-version/README.md | 31 + .../goproxy-jquery-version/jquery1.html | 8 + .../goproxy-jquery-version/jquery2.html | 8 + .../jquery_homepage.html | 233 +++ .../goproxy-jquery-version/jquery_test.go | 118 ++ .../examples/goproxy-jquery-version/main.go | 64 + .../goproxy-jquery-version/php_man.html | 323 ++++ .../goproxy-jquery-version/w3schools.html | 1610 +++++++++++++++++ .../goproxy-no-reddit-at-worktime/README.md | 21 + .../goproxy-no-reddit-at-worktime/noreddit.go | 25 + .../goproxy-sokeepalive/sokeepalive.go | 25 + .../examples/goproxy-sslstrip/sslstrip.go | 24 + .../goproxy/examples/goproxy-stats/README.md | 43 + .../goproxy/examples/goproxy-stats/main.go | 66 + .../examples/goproxy-transparent/README.md | 17 + .../examples/goproxy-transparent/proxy.sh | 29 + .../goproxy-transparent/transparent.go | 148 ++ .../goproxy-upside-down-ternet/main.go | 26 + .../examples/goproxy-yui-minify/yui.go | 91 + .../elazarl/goproxy/ext/auth/basic.go | 76 + .../elazarl/goproxy/ext/auth/basic_test.go | 175 ++ .../elazarl/goproxy/ext/html/cp1255.html | 585 ++++++ .../elazarl/goproxy/ext/html/cp1255.txt | 1 + .../elazarl/goproxy/ext/html/html.go | 104 ++ .../elazarl/goproxy/ext/html/html_test.go | 60 + .../elazarl/goproxy/ext/image/image.go | 78 + .../src/github.com/elazarl/goproxy/https.go | 366 ++++ .../src/github.com/elazarl/goproxy/key.pem | 15 + .../src/github.com/elazarl/goproxy/proxy.go | 162 ++ .../github.com/elazarl/goproxy/proxy_test.go | 767 ++++++++ .../goproxy/regretable/regretreader.go | 97 + .../goproxy/regretable/regretreader_test.go | 174 ++ .../github.com/elazarl/goproxy/responses.go | 38 + .../src/github.com/elazarl/goproxy/signer.go | 87 + .../github.com/elazarl/goproxy/signer_test.go | 87 + .../elazarl/goproxy/test_data/baby.jpg | Bin 0 -> 2571 bytes .../elazarl/goproxy/test_data/football.png | Bin 0 -> 3712 bytes .../elazarl/goproxy/test_data/panda.png | Bin 0 -> 11226 bytes .../elazarl/goproxy/transport/roundtripper.go | 19 + .../elazarl/goproxy/transport/transport.go | 789 ++++++++ .../elazarl/goproxy/transport/util.go | 15 + test/images/goproxy/.gitignore | 1 + test/images/goproxy/Dockerfile | 17 + test/images/goproxy/Makefile | 15 + test/images/goproxy/goproxy.go | 30 + 64 files changed, 8110 insertions(+) create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/.gitignore create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/LICENSE create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/README.md create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/actions.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/all.bash create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ca.pem create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/certs.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/chunked.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor_test.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ctx.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/dispatcher.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/doc.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/README.md create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/main.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-eavesdropper/main.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/README.md create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/httpdump.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/README.md create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery1.html create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery2.html create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_homepage.html create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_test.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/main.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/php_man.html create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/w3schools.html create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/README.md create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/noreddit.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sokeepalive/sokeepalive.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sslstrip/sslstrip.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/README.md create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/main.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/README.md create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/proxy.sh create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/transparent.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-upside-down-ternet/main.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-yui-minify/yui.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic_test.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.html create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.txt create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html_test.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/ext/image/image.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/https.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/key.pem create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/proxy.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/proxy_test.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader_test.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/responses.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/signer.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/signer_test.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/test_data/baby.jpg create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/test_data/football.png create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/test_data/panda.png create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/transport/roundtripper.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/transport/transport.go create mode 100644 Godeps/_workspace/src/github.com/elazarl/goproxy/transport/util.go create mode 100644 test/images/goproxy/.gitignore create mode 100644 test/images/goproxy/Dockerfile create mode 100644 test/images/goproxy/Makefile create mode 100644 test/images/goproxy/goproxy.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 8b9eac7b74fb..aab9183d331c 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -232,6 +232,11 @@ "ImportPath": "github.com/elazarl/go-bindata-assetfs", "Rev": "3dcc96556217539f50599357fb481ac0dc7439b9" }, + { + "ImportPath": "github.com/elazarl/goproxy", + "Comment": "v1.0-66-g07b16b6", + "Rev": "07b16b6e30fcac0ad8c0435548e743bcf2ca7e92" + }, { "ImportPath": "github.com/emicklei/go-restful", "Comment": "v1.1.3-98-g1f9a0ee", diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/.gitignore b/Godeps/_workspace/src/github.com/elazarl/goproxy/.gitignore new file mode 100644 index 000000000000..1005f6f1ecd6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/.gitignore @@ -0,0 +1,2 @@ +bin +*.swp diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/LICENSE b/Godeps/_workspace/src/github.com/elazarl/goproxy/LICENSE new file mode 100644 index 000000000000..2067e567c9fe --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 Elazar Leibovich. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Elazar Leibovich. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/README.md b/Godeps/_workspace/src/github.com/elazarl/goproxy/README.md new file mode 100644 index 000000000000..e94fef9ac4e1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/README.md @@ -0,0 +1,115 @@ +# Introduction + +[![Join the chat at https://gitter.im/elazarl/goproxy](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/elazarl/goproxy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +Package goproxy provides a customizable HTTP proxy library for Go (golang), + +It supports regular HTTP proxy, HTTPS through CONNECT, and "hijacking" HTTPS +connection using "Man in the Middle" style attack. + +The intent of the proxy, is to be usable with reasonable amount of traffic +yet, customizable and programable. + +The proxy itself is simply a `net/http` handler. + +In order to use goproxy, one should set their browser to use goproxy as an HTTP +proxy. Here is how you do that [in Chrome](https://support.google.com/chrome/answer/96815?hl=en) +and [in Firefox](http://www.wikihow.com/Enter-Proxy-Settings-in-Firefox). + +For example, the URL you should use as proxy when running `./bin/basic` is +`localhost:8080`, as this is the default binding for the basic proxy. + +## Mailing List + +New features would be discussed on the [mailing list](https://groups.google.com/forum/#!forum/goproxy-dev) +before their development. + +## Latest Stable Release + +Get the latest goproxy from `gopkg.in/elazarl/goproxy.v1`. + +# Why not Fiddler2? + +Fiddler is an excellent software with similar intent. However, Fiddler is not +as customable as goproxy intend to be. The main difference is, Fiddler is not +intended to be used as a real proxy. + +A possible use case that suits goproxy but +not Fiddler, is, gathering statisitics on page load times for a certain website over a week. +With goproxy you could ask all your users to set their proxy to a dedicated machine running a +goproxy server. Fiddler is a GUI app not designed to be ran like a server for multiple users. + +# A taste of goproxy + +To get a taste of `goproxy`, a basic HTTP/HTTPS transparent proxy + + + import ( + "github.com/elazarl/goproxy" + "log" + "net/http" + ) + + func main() { + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + log.Fatal(http.ListenAndServe(":8080", proxy)) + } + + +This line will add `X-GoProxy: yxorPoG-X` header to all requests sent through the proxy + + proxy.OnRequest().DoFunc( + func(r *http.Request,ctx *goproxy.ProxyCtx)(*http.Request,*http.Response) { + r.Header.Set("X-GoProxy","yxorPoG-X") + return r,nil + }) + +`DoFunc` will process all incoming requests to the proxy. It will add a header to the request +and return it. The proxy will send the modified request. + +Note that we returned nil value as the response. Have we returned a response, goproxy would +have discarded the request and sent the new response to the client. + +In order to refuse connections to reddit at work time + + proxy.OnRequest(goproxy.DstHostIs("www.reddit.com")).DoFunc( + func(r *http.Request,ctx *goproxy.ProxyCtx)(*http.Request,*http.Response) { + if h,_,_ := time.Now().Clock(); h >= 8 && h <= 17 { + return r,goproxy.NewResponse(r, + goproxy.ContentTypeText,http.StatusForbidden, + "Don't waste your time!") + } + return r,nil + }) + +`DstHostIs` returns a `ReqCondition`, that is a function receiving a `Request` and returning a boolean +we will only process requests that matches the condition. `DstHostIs("www.reddit.com")` will return +a `ReqCondition` accepting only requests directed to "www.reddit.com". + +`DoFunc` will recieve a function that will preprocess the request. We can change the request, or +return a response. If the time is between 8:00am and 17:00pm, we will neglect the request, and +return a precanned text response saying "do not waste your time". + +See additional examples in the examples directory. + +# What's New + + 1. Ability to `Hijack` CONNECT requests. See +[the eavesdropper example](https://github.com/elazarl/goproxy/blob/master/examples/goproxy-eavesdropper/main.go#L27) +2. Transparent proxy support for http/https including MITM certificate generation for TLS. See the [transparent example.](https://github.com/elazarl/goproxy/tree/master/examples/goproxy-transparent) + +# License + +I put the software temporarily under the Go-compatible BSD license, +if this prevents someone from using the software, do let mee know and I'll consider changing it. + +At any rate, user feedback is very important for me, so I'll be delighted to know if you're using this package. + +# Beta Software + +I've received a positive feedback from a few people who use goproxy in production settings. +I believe it is good enough for usage. + +I'll try to keep reasonable backwards compatability. In case of a major API change, +I'll change the import path. diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/actions.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/actions.go new file mode 100644 index 000000000000..e1a3e7ff17e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/actions.go @@ -0,0 +1,57 @@ +package goproxy + +import "net/http" + +// ReqHandler will "tamper" with the request coming to the proxy server +// If Handle returns req,nil the proxy will send the returned request +// to the destination server. If it returns nil,resp the proxy will +// skip sending any requests, and will simply return the response `resp` +// to the client. +type ReqHandler interface { + Handle(req *http.Request, ctx *ProxyCtx) (*http.Request, *http.Response) +} + +// A wrapper that would convert a function to a ReqHandler interface type +type FuncReqHandler func(req *http.Request, ctx *ProxyCtx) (*http.Request, *http.Response) + +// FuncReqHandler.Handle(req,ctx) <=> FuncReqHandler(req,ctx) +func (f FuncReqHandler) Handle(req *http.Request, ctx *ProxyCtx) (*http.Request, *http.Response) { + return f(req, ctx) +} + +// after the proxy have sent the request to the destination server, it will +// "filter" the response through the RespHandlers it has. +// The proxy server will send to the client the response returned by the RespHandler. +// In case of error, resp will be nil, and ctx.RoundTrip.Error will contain the error +type RespHandler interface { + Handle(resp *http.Response, ctx *ProxyCtx) *http.Response +} + +// A wrapper that would convert a function to a RespHandler interface type +type FuncRespHandler func(resp *http.Response, ctx *ProxyCtx) *http.Response + +// FuncRespHandler.Handle(req,ctx) <=> FuncRespHandler(req,ctx) +func (f FuncRespHandler) Handle(resp *http.Response, ctx *ProxyCtx) *http.Response { + return f(resp, ctx) +} + +// When a client send a CONNECT request to a host, the request is filtered through +// all the HttpsHandlers the proxy has, and if one returns true, the connection is +// sniffed using Man in the Middle attack. +// That is, the proxy will create a TLS connection with the client, another TLS +// connection with the destination the client wished to connect to, and would +// send back and forth all messages from the server to the client and vice versa. +// The request and responses sent in this Man In the Middle channel are filtered +// through the usual flow (request and response filtered through the ReqHandlers +// and RespHandlers) +type HttpsHandler interface { + HandleConnect(req string, ctx *ProxyCtx) (*ConnectAction, string) +} + +// A wrapper that would convert a function to a HttpsHandler interface type +type FuncHttpsHandler func(host string, ctx *ProxyCtx) (*ConnectAction, string) + +// FuncHttpsHandler should implement the RespHandler interface +func (f FuncHttpsHandler) HandleConnect(host string, ctx *ProxyCtx) (*ConnectAction, string) { + return f(host, ctx) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/all.bash b/Godeps/_workspace/src/github.com/elazarl/goproxy/all.bash new file mode 100644 index 000000000000..6503e73dc929 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/all.bash @@ -0,0 +1,15 @@ +#!/bin/bash + +go test || exit +for action in $@; do go $action; done + +mkdir -p bin +find regretable examples/* ext/* -maxdepth 0 -type d | while read d; do + (cd $d + go build -o ../../bin/$(basename $d) + find *_test.go -maxdepth 0 2>/dev/null|while read f;do + for action in $@; do go $action; done + go test + break + done) +done diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ca.pem b/Godeps/_workspace/src/github.com/elazarl/goproxy/ca.pem new file mode 100644 index 000000000000..f13842493227 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ca.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICSjCCAbWgAwIBAgIBADALBgkqhkiG9w0BAQUwSjEjMCEGA1UEChMaZ2l0aHVi +LmNvbS9lbGF6YXJsL2dvcHJveHkxIzAhBgNVBAMTGmdpdGh1Yi5jb20vZWxhemFy +bC9nb3Byb3h5MB4XDTAwMDEwMTAwMDAwMFoXDTQ5MTIzMTIzNTk1OVowSjEjMCEG +A1UEChMaZ2l0aHViLmNvbS9lbGF6YXJsL2dvcHJveHkxIzAhBgNVBAMTGmdpdGh1 +Yi5jb20vZWxhemFybC9nb3Byb3h5MIGdMAsGCSqGSIb3DQEBAQOBjQAwgYkCgYEA +vz9BbCaJjxs73Tvcq3leP32hAGerQ1RgvlZ68Z4nZmoVHfl+2Nr/m0dmW+GdOfpT +cs/KzfJjYGr/84x524fiuR8GdZ0HOtXJzyF5seoWnbBIuyr1PbEpgRhGQMqqOUuj +YExeLbfNHPIoJ8XZ1Vzyv3YxjbmjWA+S/uOe9HWtDbMCAwEAAaNGMEQwDgYDVR0P +AQH/BAQDAgCkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8w +DAYDVR0RBAUwA4IBKjALBgkqhkiG9w0BAQUDgYEAIcL8huSmGMompNujsvePTUnM +oEUKtX4Eh/+s+DSfV/TyI0I+3GiPpLplEgFWuoBIJGios0r1dKh5N0TGjxX/RmGm +qo7E4jjJuo8Gs5U8/fgThZmshax2lwLtbRNwhvUVr65GdahLsZz8I+hySLuatVvR +qHHq/FQORIiNyNpq/Hg= +-----END CERTIFICATE----- diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/certs.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/certs.go new file mode 100644 index 000000000000..8da2e6240a9a --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/certs.go @@ -0,0 +1,56 @@ +package goproxy + +import ( + "crypto/tls" + "crypto/x509" +) + +func init() { + if goproxyCaErr != nil { + panic("Error parsing builtin CA " + goproxyCaErr.Error()) + } + var err error + if GoproxyCa.Leaf, err = x509.ParseCertificate(GoproxyCa.Certificate[0]); err != nil { + panic("Error parsing builtin CA " + err.Error()) + } +} + +var tlsClientSkipVerify = &tls.Config{InsecureSkipVerify: true} + +var defaultTLSConfig = &tls.Config{ + InsecureSkipVerify: true, +} + +var CA_CERT = []byte(`-----BEGIN CERTIFICATE----- +MIICSjCCAbWgAwIBAgIBADALBgkqhkiG9w0BAQUwSjEjMCEGA1UEChMaZ2l0aHVi +LmNvbS9lbGF6YXJsL2dvcHJveHkxIzAhBgNVBAMTGmdpdGh1Yi5jb20vZWxhemFy +bC9nb3Byb3h5MB4XDTAwMDEwMTAwMDAwMFoXDTQ5MTIzMTIzNTk1OVowSjEjMCEG +A1UEChMaZ2l0aHViLmNvbS9lbGF6YXJsL2dvcHJveHkxIzAhBgNVBAMTGmdpdGh1 +Yi5jb20vZWxhemFybC9nb3Byb3h5MIGdMAsGCSqGSIb3DQEBAQOBjQAwgYkCgYEA +vz9BbCaJjxs73Tvcq3leP32hAGerQ1RgvlZ68Z4nZmoVHfl+2Nr/m0dmW+GdOfpT +cs/KzfJjYGr/84x524fiuR8GdZ0HOtXJzyF5seoWnbBIuyr1PbEpgRhGQMqqOUuj +YExeLbfNHPIoJ8XZ1Vzyv3YxjbmjWA+S/uOe9HWtDbMCAwEAAaNGMEQwDgYDVR0P +AQH/BAQDAgCkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8w +DAYDVR0RBAUwA4IBKjALBgkqhkiG9w0BAQUDgYEAIcL8huSmGMompNujsvePTUnM +oEUKtX4Eh/+s+DSfV/TyI0I+3GiPpLplEgFWuoBIJGios0r1dKh5N0TGjxX/RmGm +qo7E4jjJuo8Gs5U8/fgThZmshax2lwLtbRNwhvUVr65GdahLsZz8I+hySLuatVvR +qHHq/FQORIiNyNpq/Hg= +-----END CERTIFICATE-----`) + +var CA_KEY = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQC/P0FsJomPGzvdO9yreV4/faEAZ6tDVGC+VnrxnidmahUd+X7Y +2v+bR2Zb4Z05+lNyz8rN8mNgav/zjHnbh+K5HwZ1nQc61cnPIXmx6hadsEi7KvU9 +sSmBGEZAyqo5S6NgTF4tt80c8ignxdnVXPK/djGNuaNYD5L+4570da0NswIDAQAB +AoGBALzIv1b4D7ARTR3NOr6V9wArjiOtMjUrdLhO+9vIp9IEA8ZsA9gjDlCEwbkP +VDnoLjnWfraff5Os6+3JjHy1fYpUiCdnk2XA6iJSL1XWKQZPt3wOunxP4lalDgED +QTRReFbA/y/Z4kSfTXpVj68ytcvSRW/N7q5/qRtbN9804jpBAkEA0s6lvH2btSLA +mcEdwhs7zAslLbdld7rvfUeP82gPPk0S6yUqTNyikqshM9AwAktHY7WvYdKl+ghZ +HTxKVC4DoQJBAOg/IAW5RbXknP+Lf7AVtBgw3E+Yfa3mcdLySe8hjxxyZq825Zmu +Rt5Qj4Lw6ifSFNy4kiiSpE/ZCukYvUXGENMCQFkPxSWlS6tzSzuqQxBGwTSrYMG3 +wb6b06JyIXcMd6Qym9OMmBpw/J5KfnSNeDr/4uFVWQtTG5xO+pdHaX+3EQECQQDl +qcbY4iX1gWVfr2tNjajSYz751yoxVbkpiT9joiQLVXYFvpu+JYEfRzsjmWl0h2Lq +AftG8/xYmaEYcMZ6wSrRAkBUwiom98/8wZVlB6qbwhU1EKDFANvICGSWMIhPx3v7 +MJqTIj4uJhte2/uyVvZ6DC6noWYgy+kLgqG0S97tUEG8 +-----END RSA PRIVATE KEY-----`) + +var GoproxyCa, goproxyCaErr = tls.X509KeyPair(CA_CERT, CA_KEY) diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/chunked.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/chunked.go new file mode 100644 index 000000000000..83654f658630 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/chunked.go @@ -0,0 +1,59 @@ +// Taken from $GOROOT/src/pkg/net/http/chunked +// needed to write https responses to client. +package goproxy + +import ( + "io" + "strconv" +) + +// newChunkedWriter returns a new chunkedWriter that translates writes into HTTP +// "chunked" format before writing them to w. Closing the returned chunkedWriter +// sends the final 0-length chunk that marks the end of the stream. +// +// newChunkedWriter is not needed by normal applications. The http +// package adds chunking automatically if handlers don't set a +// Content-Length header. Using newChunkedWriter inside a handler +// would result in double chunking or chunking with a Content-Length +// length, both of which are wrong. +func newChunkedWriter(w io.Writer) io.WriteCloser { + return &chunkedWriter{w} +} + +// Writing to chunkedWriter translates to writing in HTTP chunked Transfer +// Encoding wire format to the underlying Wire chunkedWriter. +type chunkedWriter struct { + Wire io.Writer +} + +// Write the contents of data as one chunk to Wire. +// NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has +// a bug since it does not check for success of io.WriteString +func (cw *chunkedWriter) Write(data []byte) (n int, err error) { + + // Don't send 0-length data. It looks like EOF for chunked encoding. + if len(data) == 0 { + return 0, nil + } + + head := strconv.FormatInt(int64(len(data)), 16) + "\r\n" + + if _, err = io.WriteString(cw.Wire, head); err != nil { + return 0, err + } + if n, err = cw.Wire.Write(data); err != nil { + return + } + if n != len(data) { + err = io.ErrShortWrite + return + } + _, err = io.WriteString(cw.Wire, "\r\n") + + return +} + +func (cw *chunkedWriter) Close() error { + _, err := io.WriteString(cw.Wire, "0\r\n") + return err +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor.go new file mode 100644 index 000000000000..494e7a4fedfe --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor.go @@ -0,0 +1,68 @@ +package goproxy + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "errors" +) + +type CounterEncryptorRand struct { + cipher cipher.Block + counter []byte + rand []byte + ix int +} + +func NewCounterEncryptorRandFromKey(key interface{}, seed []byte) (r CounterEncryptorRand, err error) { + var keyBytes []byte + switch key := key.(type) { + case *rsa.PrivateKey: + keyBytes = x509.MarshalPKCS1PrivateKey(key) + default: + err = errors.New("only RSA keys supported") + return + } + h := sha256.New() + if r.cipher, err = aes.NewCipher(h.Sum(keyBytes)[:aes.BlockSize]); err != nil { + return + } + r.counter = make([]byte, r.cipher.BlockSize()) + if seed != nil { + copy(r.counter, h.Sum(seed)[:r.cipher.BlockSize()]) + } + r.rand = make([]byte, r.cipher.BlockSize()) + r.ix = len(r.rand) + return +} + +func (c *CounterEncryptorRand) Seed(b []byte) { + if len(b) != len(c.counter) { + panic("SetCounter: wrong counter size") + } + copy(c.counter, b) +} + +func (c *CounterEncryptorRand) refill() { + c.cipher.Encrypt(c.rand, c.counter) + for i := 0; i < len(c.counter); i++ { + if c.counter[i]++; c.counter[i] != 0 { + break + } + } + c.ix = 0 +} + +func (c *CounterEncryptorRand) Read(b []byte) (n int, err error) { + if c.ix == len(c.rand) { + c.refill() + } + if n = len(c.rand) - c.ix; n > len(b) { + n = len(b) + } + copy(b, c.rand[c.ix:c.ix+n]) + c.ix += n + return +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor_test.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor_test.go new file mode 100644 index 000000000000..12b31e16f481 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/counterecryptor_test.go @@ -0,0 +1,99 @@ +package goproxy_test + +import ( + "bytes" + "crypto/rsa" + "encoding/binary" + "github.com/elazarl/goproxy" + "io" + "math" + "math/rand" + "testing" +) + +type RandSeedReader struct { + r rand.Rand +} + +func (r *RandSeedReader) Read(b []byte) (n int, err error) { + for i := range b { + b[i] = byte(r.r.Int() & 0xFF) + } + return len(b), nil +} + +func TestCounterEncDifferentConsecutive(t *testing.T) { + k, err := rsa.GenerateKey(&RandSeedReader{*rand.New(rand.NewSource(0xFF43109))}, 128) + fatalOnErr(err, "rsa.GenerateKey", t) + c, err := goproxy.NewCounterEncryptorRandFromKey(k, []byte("the quick brown fox run over the lazy dog")) + fatalOnErr(err, "NewCounterEncryptorRandFromKey", t) + for i := 0; i < 100*1000; i++ { + var a, b int64 + binary.Read(&c, binary.BigEndian, &a) + binary.Read(&c, binary.BigEndian, &b) + if a == b { + t.Fatal("two consecutive equal int64", a, b) + } + } +} + +func TestCounterEncIdenticalStreams(t *testing.T) { + k, err := rsa.GenerateKey(&RandSeedReader{*rand.New(rand.NewSource(0xFF43109))}, 128) + fatalOnErr(err, "rsa.GenerateKey", t) + c1, err := goproxy.NewCounterEncryptorRandFromKey(k, []byte("the quick brown fox run over the lazy dog")) + fatalOnErr(err, "NewCounterEncryptorRandFromKey", t) + c2, err := goproxy.NewCounterEncryptorRandFromKey(k, []byte("the quick brown fox run over the lazy dog")) + fatalOnErr(err, "NewCounterEncryptorRandFromKey", t) + nout := 1000 + out1, out2 := make([]byte, nout), make([]byte, nout) + io.ReadFull(&c1, out1) + tmp := out2[:] + rand.Seed(0xFF43109) + for len(tmp) > 0 { + n := 1 + rand.Intn(256) + if n > len(tmp) { + n = len(tmp) + } + n, err := c2.Read(tmp[:n]) + fatalOnErr(err, "CounterEncryptorRand.Read", t) + tmp = tmp[n:] + } + if !bytes.Equal(out1, out2) { + t.Error("identical CSPRNG does not produce the same output") + } +} + +func stddev(data []int) float64 { + var sum, sum_sqr float64 = 0, 0 + for _, h := range data { + sum += float64(h) + sum_sqr += float64(h) * float64(h) + } + n := float64(len(data)) + variance := (sum_sqr - ((sum * sum) / n)) / (n - 1) + return math.Sqrt(variance) +} + +func TestCounterEncStreamHistogram(t *testing.T) { + k, err := rsa.GenerateKey(&RandSeedReader{*rand.New(rand.NewSource(0xFF43109))}, 128) + fatalOnErr(err, "rsa.GenerateKey", t) + c, err := goproxy.NewCounterEncryptorRandFromKey(k, []byte("the quick brown fox run over the lazy dog")) + fatalOnErr(err, "NewCounterEncryptorRandFromKey", t) + nout := 100 * 1000 + out := make([]byte, nout) + io.ReadFull(&c, out) + refhist := make([]int, 256) + for i := 0; i < nout; i++ { + refhist[rand.Intn(256)]++ + } + hist := make([]int, 256) + for _, b := range out { + hist[int(b)]++ + } + refstddev, stddev := stddev(refhist), stddev(hist) + // due to lack of time, I guestimate + t.Logf("ref:%v - act:%v = %v", refstddev, stddev, math.Abs(refstddev-stddev)) + if math.Abs(refstddev-stddev) >= 1 { + t.Errorf("stddev of ref histogram different than regular PRNG: %v %v", refstddev, stddev) + } +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ctx.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/ctx.go new file mode 100644 index 000000000000..95bfd8004381 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ctx.go @@ -0,0 +1,87 @@ +package goproxy + +import ( + "net/http" + "regexp" +) + +// ProxyCtx is the Proxy context, contains useful information about every request. It is passed to +// every user function. Also used as a logger. +type ProxyCtx struct { + // Will contain the client request from the proxy + Req *http.Request + // Will contain the remote server's response (if available. nil if the request wasn't send yet) + Resp *http.Response + RoundTripper RoundTripper + // will contain the recent error that occured while trying to send receive or parse traffic + Error error + // A handle for the user to keep data in the context, from the call of ReqHandler to the + // call of RespHandler + UserData interface{} + // Will connect a request to a response + Session int64 + proxy *ProxyHttpServer +} + +type RoundTripper interface { + RoundTrip(req *http.Request, ctx *ProxyCtx) (*http.Response, error) +} + +type RoundTripperFunc func(req *http.Request, ctx *ProxyCtx) (*http.Response, error) + +func (f RoundTripperFunc) RoundTrip(req *http.Request, ctx *ProxyCtx) (*http.Response, error) { + return f(req, ctx) +} + +func (ctx *ProxyCtx) RoundTrip(req *http.Request) (*http.Response, error) { + if ctx.RoundTripper != nil { + return ctx.RoundTripper.RoundTrip(req, ctx) + } + return ctx.proxy.Tr.RoundTrip(req) +} + +func (ctx *ProxyCtx) printf(msg string, argv ...interface{}) { + ctx.proxy.Logger.Printf("[%03d] "+msg+"\n", append([]interface{}{ctx.Session & 0xFF}, argv...)...) +} + +// Logf prints a message to the proxy's log. Should be used in a ProxyHttpServer's filter +// This message will be printed only if the Verbose field of the ProxyHttpServer is set to true +// +// proxy.OnRequest().DoFunc(func(r *http.Request,ctx *goproxy.ProxyCtx) (*http.Request, *http.Response){ +// nr := atomic.AddInt32(&counter,1) +// ctx.Printf("So far %d requests",nr) +// return r, nil +// }) +func (ctx *ProxyCtx) Logf(msg string, argv ...interface{}) { + if ctx.proxy.Verbose { + ctx.printf("INFO: "+msg, argv...) + } +} + +// Warnf prints a message to the proxy's log. Should be used in a ProxyHttpServer's filter +// This message will always be printed. +// +// proxy.OnRequest().DoFunc(func(r *http.Request,ctx *goproxy.ProxyCtx) (*http.Request, *http.Response){ +// f,err := os.OpenFile(cachedContent) +// if err != nil { +// ctx.Warnf("error open file %v: %v",cachedContent,err) +// return r, nil +// } +// return r, nil +// }) +func (ctx *ProxyCtx) Warnf(msg string, argv ...interface{}) { + ctx.printf("WARN: "+msg, argv...) +} + +var charsetFinder = regexp.MustCompile("charset=([^ ;]*)") + +// Will try to infer the character set of the request from the headers. +// Returns the empty string if we don't know which character set it used. +// Currently it will look for charset= in the Content-Type header of the request. +func (ctx *ProxyCtx) Charset() string { + charsets := charsetFinder.FindStringSubmatch(ctx.Resp.Header.Get("Content-Type")) + if charsets == nil { + return "" + } + return charsets[1] +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/dispatcher.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/dispatcher.go new file mode 100644 index 000000000000..69219b365e3b --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/dispatcher.go @@ -0,0 +1,320 @@ +package goproxy + +import ( + "bytes" + "io/ioutil" + "net" + "net/http" + "regexp" + "strings" +) + +// ReqCondition.HandleReq will decide whether or not to use the ReqHandler on an HTTP request +// before sending it to the remote server +type ReqCondition interface { + RespCondition + HandleReq(req *http.Request, ctx *ProxyCtx) bool +} + +// RespCondition.HandleReq will decide whether or not to use the RespHandler on an HTTP response +// before sending it to the proxy client. Note that resp might be nil, in case there was an +// error sending the request. +type RespCondition interface { + HandleResp(resp *http.Response, ctx *ProxyCtx) bool +} + +// ReqConditionFunc.HandleReq(req,ctx) <=> ReqConditionFunc(req,ctx) +type ReqConditionFunc func(req *http.Request, ctx *ProxyCtx) bool + +// RespConditionFunc.HandleResp(resp,ctx) <=> RespConditionFunc(resp,ctx) +type RespConditionFunc func(resp *http.Response, ctx *ProxyCtx) bool + +func (c ReqConditionFunc) HandleReq(req *http.Request, ctx *ProxyCtx) bool { + return c(req, ctx) +} + +// ReqConditionFunc cannot test responses. It only satisfies RespCondition interface so that +// to be usable as RespCondition. +func (c ReqConditionFunc) HandleResp(resp *http.Response, ctx *ProxyCtx) bool { + return c(ctx.Req, ctx) +} + +func (c RespConditionFunc) HandleResp(resp *http.Response, ctx *ProxyCtx) bool { + return c(resp, ctx) +} + +// UrlHasPrefix returns a ReqCondition checking wether the destination URL the proxy client has requested +// has the given prefix, with or without the host. +// For example UrlHasPrefix("host/x") will match requests of the form 'GET host/x', and will match +// requests to url 'http://host/x' +func UrlHasPrefix(prefix string) ReqConditionFunc { + return func(req *http.Request, ctx *ProxyCtx) bool { + return strings.HasPrefix(req.URL.Path, prefix) || + strings.HasPrefix(req.URL.Host+req.URL.Path, prefix) || + strings.HasPrefix(req.URL.Scheme+req.URL.Host+req.URL.Path, prefix) + } +} + +// UrlIs returns a ReqCondition, testing whether or not the request URL is one of the given strings +// with or without the host prefix. +// UrlIs("google.com/","foo") will match requests 'GET /' to 'google.com', requests `'GET google.com/' to +// any host, and requests of the form 'GET foo'. +func UrlIs(urls ...string) ReqConditionFunc { + urlSet := make(map[string]bool) + for _, u := range urls { + urlSet[u] = true + } + return func(req *http.Request, ctx *ProxyCtx) bool { + _, pathOk := urlSet[req.URL.Path] + _, hostAndOk := urlSet[req.URL.Host+req.URL.Path] + return pathOk || hostAndOk + } +} + +// ReqHostMatches returns a ReqCondition, testing whether the host to which the request was directed to matches +// any of the given regular expressions. +func ReqHostMatches(regexps ...*regexp.Regexp) ReqConditionFunc { + return func(req *http.Request, ctx *ProxyCtx) bool { + for _, re := range regexps { + if re.MatchString(req.Host) { + return true + } + } + return false + } +} + +// ReqHostIs returns a ReqCondition, testing whether the host to which the request is directed to equal +// to one of the given strings +func ReqHostIs(hosts ...string) ReqConditionFunc { + hostSet := make(map[string]bool) + for _, h := range hosts { + hostSet[h] = true + } + return func(req *http.Request, ctx *ProxyCtx) bool { + _, ok := hostSet[req.URL.Host] + return ok + } +} + +var localHostIpv4 = regexp.MustCompile(`127\.0\.0\.\d+`) + +// IsLocalHost checks whether the destination host is explicitly local host +// (buggy, there can be IPv6 addresses it doesn't catch) +var IsLocalHost ReqConditionFunc = func(req *http.Request, ctx *ProxyCtx) bool { + return req.URL.Host == "::1" || + req.URL.Host == "0:0:0:0:0:0:0:1" || + localHostIpv4.MatchString(req.URL.Host) || + req.URL.Host == "localhost" +} + +// UrlMatches returns a ReqCondition testing whether the destination URL +// of the request matches the given regexp, with or without prefix +func UrlMatches(re *regexp.Regexp) ReqConditionFunc { + return func(req *http.Request, ctx *ProxyCtx) bool { + return re.MatchString(req.URL.Path) || + re.MatchString(req.URL.Host+req.URL.Path) + } +} + +// DstHostIs returns a ReqCondition testing wether the host in the request url is the given string +func DstHostIs(host string) ReqConditionFunc { + return func(req *http.Request, ctx *ProxyCtx) bool { + return req.URL.Host == host + } +} + +// SrcIpIs returns a ReqCondition testing wether the source IP of the request is the given string +func SrcIpIs(ip string) ReqCondition { + return ReqConditionFunc(func(req *http.Request, ctx *ProxyCtx) bool { + return strings.HasPrefix(req.RemoteAddr, ip+":") + }) +} + +// Not returns a ReqCondition negating the given ReqCondition +func Not(r ReqCondition) ReqConditionFunc { + return func(req *http.Request, ctx *ProxyCtx) bool { + return !r.HandleReq(req, ctx) + } +} + +// ContentTypeIs returns a RespCondition testing whether the HTTP response has Content-Type header equal +// to one of the given strings. +func ContentTypeIs(typ string, types ...string) RespCondition { + types = append(types, typ) + return RespConditionFunc(func(resp *http.Response, ctx *ProxyCtx) bool { + if resp == nil { + return false + } + contentType := resp.Header.Get("Content-Type") + for _, typ := range types { + if contentType == typ || strings.HasPrefix(contentType, typ+";") { + return true + } + } + return false + }) +} + +// ProxyHttpServer.OnRequest Will return a temporary ReqProxyConds struct, aggregating the given condtions. +// You will use the ReqProxyConds struct to register a ReqHandler, that would filter +// the request, only if all the given ReqCondition matched. +// Typical usage: +// proxy.OnRequest(UrlIs("example.com/foo"),UrlMatches(regexp.MustParse(`.*\.exampl.\com\./.*`)).Do(...) +func (proxy *ProxyHttpServer) OnRequest(conds ...ReqCondition) *ReqProxyConds { + return &ReqProxyConds{proxy, conds} +} + +// ReqProxyConds aggregate ReqConditions for a ProxyHttpServer. Upon calling Do, it will register a ReqHandler that would +// handle the request if all conditions on the HTTP request are met. +type ReqProxyConds struct { + proxy *ProxyHttpServer + reqConds []ReqCondition +} + +// DoFunc is equivalent to proxy.OnRequest().Do(FuncReqHandler(f)) +func (pcond *ReqProxyConds) DoFunc(f func(req *http.Request, ctx *ProxyCtx) (*http.Request, *http.Response)) { + pcond.Do(FuncReqHandler(f)) +} + +// ReqProxyConds.Do will register the ReqHandler on the proxy, +// the ReqHandler will handle the HTTP request if all the conditions +// aggregated in the ReqProxyConds are met. Typical usage: +// proxy.OnRequest().Do(handler) // will call handler.Handle(req,ctx) on every request to the proxy +// proxy.OnRequest(cond1,cond2).Do(handler) +// // given request to the proxy, will test if cond1.HandleReq(req,ctx) && cond2.HandleReq(req,ctx) are true +// // if they are, will call handler.Handle(req,ctx) +func (pcond *ReqProxyConds) Do(h ReqHandler) { + pcond.proxy.reqHandlers = append(pcond.proxy.reqHandlers, + FuncReqHandler(func(r *http.Request, ctx *ProxyCtx) (*http.Request, *http.Response) { + for _, cond := range pcond.reqConds { + if !cond.HandleReq(r, ctx) { + return r, nil + } + } + return h.Handle(r, ctx) + })) +} + +// HandleConnect is used when proxy receives an HTTP CONNECT request, +// it'll then use the HttpsHandler to determine what should it +// do with this request. The handler returns a ConnectAction struct, the Action field in the ConnectAction +// struct returned will determine what to do with this request. ConnectAccept will simply accept the request +// forwarding all bytes from the client to the remote host, ConnectReject will close the connection with the +// client, and ConnectMitm, will assume the underlying connection is an HTTPS connection, and will use Man +// in the Middle attack to eavesdrop the connection. All regular handler will be active on this eavesdropped +// connection. +// The ConnectAction struct contains possible tlsConfig that will be used for eavesdropping. If nil, the proxy +// will use the default tls configuration. +// proxy.OnRequest().HandleConnect(goproxy.AlwaysReject) // rejects all CONNECT requests +func (pcond *ReqProxyConds) HandleConnect(h HttpsHandler) { + pcond.proxy.httpsHandlers = append(pcond.proxy.httpsHandlers, + FuncHttpsHandler(func(host string, ctx *ProxyCtx) (*ConnectAction, string) { + for _, cond := range pcond.reqConds { + if !cond.HandleReq(ctx.Req, ctx) { + return nil, "" + } + } + return h.HandleConnect(host, ctx) + })) +} + +// HandleConnectFunc is equivalent to HandleConnect, +// for example, accepting CONNECT request if they contain a password in header +// io.WriteString(h,password) +// passHash := h.Sum(nil) +// proxy.OnRequest().HandleConnectFunc(func(host string, ctx *ProxyCtx) (*ConnectAction, string) { +// c := sha1.New() +// io.WriteString(c,ctx.Req.Header.Get("X-GoProxy-Auth")) +// if c.Sum(nil) == passHash { +// return OkConnect, host +// } +// return RejectConnect, host +// }) +func (pcond *ReqProxyConds) HandleConnectFunc(f func(host string, ctx *ProxyCtx) (*ConnectAction, string)) { + pcond.HandleConnect(FuncHttpsHandler(f)) +} + +func (pcond *ReqProxyConds) HijackConnect(f func(req *http.Request, client net.Conn, ctx *ProxyCtx)) { + pcond.proxy.httpsHandlers = append(pcond.proxy.httpsHandlers, + FuncHttpsHandler(func(host string, ctx *ProxyCtx) (*ConnectAction, string) { + for _, cond := range pcond.reqConds { + if !cond.HandleReq(ctx.Req, ctx) { + return nil, "" + } + } + return &ConnectAction{Action: ConnectHijack, Hijack: f}, host + })) +} + +// ProxyConds is used to aggregate RespConditions for a ProxyHttpServer. +// Upon calling ProxyConds.Do, it will register a RespHandler that would +// handle the HTTP response from remote server if all conditions on the HTTP response are met. +type ProxyConds struct { + proxy *ProxyHttpServer + reqConds []ReqCondition + respCond []RespCondition +} + +// ProxyConds.DoFunc is equivalent to proxy.OnResponse().Do(FuncRespHandler(f)) +func (pcond *ProxyConds) DoFunc(f func(resp *http.Response, ctx *ProxyCtx) *http.Response) { + pcond.Do(FuncRespHandler(f)) +} + +// ProxyConds.Do will register the RespHandler on the proxy, h.Handle(resp,ctx) will be called on every +// request that matches the conditions aggregated in pcond. +func (pcond *ProxyConds) Do(h RespHandler) { + pcond.proxy.respHandlers = append(pcond.proxy.respHandlers, + FuncRespHandler(func(resp *http.Response, ctx *ProxyCtx) *http.Response { + for _, cond := range pcond.reqConds { + if !cond.HandleReq(ctx.Req, ctx) { + return resp + } + } + for _, cond := range pcond.respCond { + if !cond.HandleResp(resp, ctx) { + return resp + } + } + return h.Handle(resp, ctx) + })) +} + +// OnResponse is used when adding a response-filter to the HTTP proxy, usual pattern is +// proxy.OnResponse(cond1,cond2).Do(handler) // handler.Handle(resp,ctx) will be used +// // if cond1.HandleResp(resp) && cond2.HandleResp(resp) +func (proxy *ProxyHttpServer) OnResponse(conds ...RespCondition) *ProxyConds { + return &ProxyConds{proxy, make([]ReqCondition, 0), conds} +} + +// AlwaysMitm is a HttpsHandler that always eavesdrop https connections, for example to +// eavesdrop all https connections to www.google.com, we can use +// proxy.OnRequest(goproxy.ReqHostIs("www.google.com")).HandleConnect(goproxy.AlwaysMitm) +var AlwaysMitm FuncHttpsHandler = func(host string, ctx *ProxyCtx) (*ConnectAction, string) { + return MitmConnect, host +} + +// AlwaysReject is a HttpsHandler that drops any CONNECT request, for example, this code will disallow +// connections to hosts on any other port than 443 +// proxy.OnRequest(goproxy.Not(goproxy.ReqHostMatches(regexp.MustCompile(":443$"))). +// HandleConnect(goproxy.AlwaysReject) +var AlwaysReject FuncHttpsHandler = func(host string, ctx *ProxyCtx) (*ConnectAction, string) { + return RejectConnect, host +} + +// HandleBytes will return a RespHandler that read the entire body of the request +// to a byte array in memory, would run the user supplied f function on the byte arra, +// and will replace the body of the original response with the resulting byte array. +func HandleBytes(f func(b []byte, ctx *ProxyCtx) []byte) RespHandler { + return FuncRespHandler(func(resp *http.Response, ctx *ProxyCtx) *http.Response { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + ctx.Warnf("Cannot read response %s", err) + return resp + } + resp.Body.Close() + + resp.Body = ioutil.NopCloser(bytes.NewBuffer(f(b, ctx))) + return resp + }) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/doc.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/doc.go new file mode 100644 index 000000000000..50aaa71f80ce --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/doc.go @@ -0,0 +1,100 @@ +/* +Package goproxy provides a customizable HTTP proxy, +supporting hijacking HTTPS connection. + +The intent of the proxy, is to be usable with reasonable amount of traffic +yet, customizable and programable. + +The proxy itself is simply an `net/http` handler. + +Typical usage is + + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest(..conditions..).Do(..requesthandler..) + proxy.OnRequest(..conditions..).DoFunc(..requesthandlerFunction..) + proxy.OnResponse(..conditions..).Do(..responesHandler..) + proxy.OnResponse(..conditions..).DoFunc(..responesHandlerFunction..) + http.ListenAndServe(":8080", proxy) + +Adding a header to each request + + proxy.OnRequest().DoFunc(func(r *http.Request,ctx *goproxy.ProxyCtx) (*http.Request, *http.Response){ + r.Header.Set("X-GoProxy","1") + return r, nil + }) + +Note that the function is called before the proxy sends the request to the server + +For printing the content type of all incoming responses + + proxy.OnResponse().DoFunc(func(r *http.Response, ctx *goproxy.ProxyCtx)*http.Response{ + println(ctx.Req.Host,"->",r.Header.Get("Content-Type")) + return r + }) + +note that we used the ProxyCtx context variable here. It contains the request +and the response (Req and Resp, Resp is nil if unavailable) of this specific client +interaction with the proxy. + +To print the content type of all responses from a certain url, we'll add a +ReqCondition to the OnResponse function: + + proxy.OnResponse(goproxy.UrlIs("golang.org/pkg")).DoFunc(func(r *http.Response, ctx *goproxy.ProxyCtx)*http.Response{ + println(ctx.Req.Host,"->",r.Header.Get("Content-Type")) + return r + }) + +We can write the condition ourselves, conditions can be set on request and on response + + var random = ReqConditionFunc(func(r *http.Request) bool { + return rand.Intn(1) == 0 + }) + var hasGoProxyHeader = RespConditionFunc(func(resp *http.Response,req *http.Request)bool { + return resp.Header.Get("X-GoProxy") != "" + }) + +Caution! If you give a RespCondition to the OnRequest function, you'll get a run time panic! It doesn't +make sense to read the response, if you still haven't got it! + +Finally, we have convenience function to throw a quick response + + proxy.OnResponse(hasGoProxyHeader).DoFunc(func(r*http.Response,ctx *goproxy.ProxyCtx)*http.Response { + r.Body.Close() + return goproxy.ForbiddenTextResponse(ctx.Req,"Can't see response with X-GoProxy header!") + }) + +we close the body of the original repsonse, and return a new 403 response with a short message. + +Example use cases: + +1. https://github.com/elazarl/goproxy/tree/master/examples/goproxy-avgsize + +To measure the average size of an Html served in your site. One can ask +all the QA team to access the website by a proxy, and the proxy will +measure the average size of all text/html responses from your host. + +2. [not yet implemented] + +All requests to your web servers should be directed through the proxy, +when the proxy will detect html pieces sent as a response to AJAX +request, it'll send a warning email. + +3. https://github.com/elazarl/goproxy/blob/master/examples/goproxy-httpdump/ + +Generate a real traffic to your website by real users using through +proxy. Record the traffic, and try it again for more real load testing. + +4. https://github.com/elazarl/goproxy/tree/master/examples/goproxy-no-reddit-at-worktime + +Will allow browsing to reddit.com between 8:00am and 17:00pm + +5. https://github.com/elazarl/goproxy/tree/master/examples/goproxy-jquery-version + +Will warn if multiple versions of jquery are used in the same domain. + +6. https://github.com/elazarl/goproxy/blob/master/examples/goproxy-upside-down-ternet/ + +Modifies image files in an HTTP response via goproxy's image extension found in ext/. + +*/ +package goproxy diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/README.md b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/README.md new file mode 100644 index 000000000000..8778f2a75b27 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/README.md @@ -0,0 +1,29 @@ +# Simple HTTP Proxy + +`goproxy-basic` starts an HTTP proxy on :8080. It only handles explicit CONNECT +requests. + +Start it in one shell: + +```sh +goproxy-basic -v +``` + +Fetch goproxy homepage in another: + +```sh +http_proxy=http://127.0.0.1:8080 wget -O - \ + http://ripper234.com/p/introducing-goproxy-light-http-proxy/ +``` + +The homepage HTML content should be displayed in the console. The proxy should +have logged the request being processed: + +```sh +2015/04/09 18:19:17 [001] INFO: Got request /p/introducing-goproxy-light-http-proxy/ ripper234.com GET http://ripper234.com/p/introducing-goproxy-light-http-proxy/ +2015/04/09 18:19:17 [001] INFO: Sending request GET http://ripper234.com/p/introducing-goproxy-light-http-proxy/ +2015/04/09 18:19:18 [001] INFO: Received response 200 OK +2015/04/09 18:19:18 [001] INFO: Copying response to client 200 OK [200] +2015/04/09 18:19:18 [001] INFO: Copied 44333 bytes to client error= +``` + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/main.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/main.go new file mode 100644 index 000000000000..22dc4a9073bc --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-basic/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "github.com/elazarl/goproxy" + "log" + "flag" + "net/http" +) + +func main() { + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + flag.Parse() + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = *verbose + log.Fatal(http.ListenAndServe(*addr, proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-eavesdropper/main.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-eavesdropper/main.go new file mode 100644 index 000000000000..9d80653be5e4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-eavesdropper/main.go @@ -0,0 +1,56 @@ +package main + +import ( + "bufio" + "flag" + "log" + "net" + "net/http" + "regexp" + + "github.com/elazarl/goproxy" +) + +func orPanic(err error) { + if err != nil { + panic(err) + } +} + +func main() { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*baidu.com$"))). + HandleConnect(goproxy.AlwaysReject) + proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))). + HandleConnect(goproxy.AlwaysMitm) + // enable curl -p for all hosts on port 80 + proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*:80$"))). + HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) { + defer func() { + if e := recover(); e != nil { + ctx.Logf("error connecting to remote: %v", e) + client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n")) + } + client.Close() + }() + clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client)) + remote, err := net.Dial("tcp", req.URL.Host) + orPanic(err) + remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote)) + for { + req, err := http.ReadRequest(clientBuf.Reader) + orPanic(err) + orPanic(req.Write(remoteBuf)) + orPanic(remoteBuf.Flush()) + resp, err := http.ReadResponse(remoteBuf.Reader, req) + orPanic(err) + orPanic(resp.Write(clientBuf.Writer)) + orPanic(clientBuf.Flush()) + } + }) + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + flag.Parse() + proxy.Verbose = *verbose + log.Fatal(http.ListenAndServe(*addr, proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/README.md b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/README.md new file mode 100644 index 000000000000..7240d8eacfe9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/README.md @@ -0,0 +1,30 @@ +# Trace HTTP Requests and Responses + +`goproxy-httpdump` starts an HTTP proxy on :8080. It handles explicit CONNECT +requests and traces them in a "db" directory created in the proxy working +directory. Each request type and headers are logged in a "log" file, while +their bodies are dumped in files prefixed with the request session identifier. + +Additionally, the example demonstrates how to: +- Log information asynchronously (see HttpLogger) +- Allow the proxy to be stopped manually while ensuring all pending requests + have been processed (in this case, logged). + +Start it in one shell: + +```sh +goproxy-httpdump +``` + +Fetch goproxy homepage in another: + +```sh +http_proxy=http://127.0.0.1:8080 wget -O - \ + http://ripper234.com/p/introducing-goproxy-light-http-proxy/ +``` + +A "db" directory should have appeared where you started the proxy, containing +two files: +- log: the request/response traces +- 1\_resp: the first response body + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/httpdump.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/httpdump.go new file mode 100644 index 000000000000..62a9b882373e --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-httpdump/httpdump.go @@ -0,0 +1,285 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "log" + "net" + "net/http" + "net/http/httputil" + "os" + "os/signal" + "path" + "sync" + "time" + + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/transport" +) + +type FileStream struct { + path string + f *os.File +} + +func NewFileStream(path string) *FileStream { + return &FileStream{path, nil} +} + +func (fs *FileStream) Write(b []byte) (nr int, err error) { + if fs.f == nil { + fs.f, err = os.Create(fs.path) + if err != nil { + return 0, err + } + } + return fs.f.Write(b) +} + +func (fs *FileStream) Close() error { + fmt.Println("Close", fs.path) + if fs.f == nil { + return errors.New("FileStream was never written into") + } + return fs.f.Close() +} + +type Meta struct { + req *http.Request + resp *http.Response + err error + t time.Time + sess int64 + bodyPath string + from string +} + +func fprintf(nr *int64, err *error, w io.Writer, pat string, a ...interface{}) { + if *err != nil { + return + } + var n int + n, *err = fmt.Fprintf(w, pat, a...) + *nr += int64(n) +} + +func write(nr *int64, err *error, w io.Writer, b []byte) { + if *err != nil { + return + } + var n int + n, *err = w.Write(b) + *nr += int64(n) +} + +func (m *Meta) WriteTo(w io.Writer) (nr int64, err error) { + if m.req != nil { + fprintf(&nr, &err, w, "Type: request\r\n") + } else if m.resp != nil { + fprintf(&nr, &err, w, "Type: response\r\n") + } + fprintf(&nr, &err, w, "ReceivedAt: %v\r\n", m.t) + fprintf(&nr, &err, w, "Session: %d\r\n", m.sess) + fprintf(&nr, &err, w, "From: %v\r\n", m.from) + if m.err != nil { + // note the empty response + fprintf(&nr, &err, w, "Error: %v\r\n\r\n\r\n\r\n", m.err) + } else if m.req != nil { + fprintf(&nr, &err, w, "\r\n") + buf, err2 := httputil.DumpRequest(m.req, false) + if err2 != nil { + return nr, err2 + } + write(&nr, &err, w, buf) + } else if m.resp != nil { + fprintf(&nr, &err, w, "\r\n") + buf, err2 := httputil.DumpResponse(m.resp, false) + if err2 != nil { + return nr, err2 + } + write(&nr, &err, w, buf) + } + return +} + +// HttpLogger is an asynchronous HTTP request/response logger. It traces +// requests and responses headers in a "log" file in logger directory and dumps +// their bodies in files prefixed with the session identifiers. +// Close it to ensure pending items are correctly logged. +type HttpLogger struct { + path string + c chan *Meta + errch chan error +} + +func NewLogger(basepath string) (*HttpLogger, error) { + f, err := os.Create(path.Join(basepath, "log")) + if err != nil { + return nil, err + } + logger := &HttpLogger{basepath, make(chan *Meta), make(chan error)} + go func() { + for m := range logger.c { + if _, err := m.WriteTo(f); err != nil { + log.Println("Can't write meta", err) + } + } + logger.errch <- f.Close() + }() + return logger, nil +} + +func (logger *HttpLogger) LogResp(resp *http.Response, ctx *goproxy.ProxyCtx) { + body := path.Join(logger.path, fmt.Sprintf("%d_resp", ctx.Session)) + from := "" + if ctx.UserData != nil { + from = ctx.UserData.(*transport.RoundTripDetails).TCPAddr.String() + } + if resp == nil { + resp = emptyResp + } else { + resp.Body = NewTeeReadCloser(resp.Body, NewFileStream(body)) + } + logger.LogMeta(&Meta{ + resp: resp, + err: ctx.Error, + t: time.Now(), + sess: ctx.Session, + from: from}) +} + +var emptyResp = &http.Response{} +var emptyReq = &http.Request{} + +func (logger *HttpLogger) LogReq(req *http.Request, ctx *goproxy.ProxyCtx) { + body := path.Join(logger.path, fmt.Sprintf("%d_req", ctx.Session)) + if req == nil { + req = emptyReq + } else { + req.Body = NewTeeReadCloser(req.Body, NewFileStream(body)) + } + logger.LogMeta(&Meta{ + req: req, + err: ctx.Error, + t: time.Now(), + sess: ctx.Session, + from: req.RemoteAddr}) +} + +func (logger *HttpLogger) LogMeta(m *Meta) { + logger.c <- m +} + +func (logger *HttpLogger) Close() error { + close(logger.c) + return <-logger.errch +} + +// TeeReadCloser extends io.TeeReader by allowing reader and writer to be +// closed. +type TeeReadCloser struct { + r io.Reader + w io.WriteCloser + c io.Closer +} + +func NewTeeReadCloser(r io.ReadCloser, w io.WriteCloser) io.ReadCloser { + return &TeeReadCloser{io.TeeReader(r, w), w, r} +} + +func (t *TeeReadCloser) Read(b []byte) (int, error) { + return t.r.Read(b) +} + +// Close attempts to close the reader and write. It returns an error if both +// failed to Close. +func (t *TeeReadCloser) Close() error { + err1 := t.c.Close() + err2 := t.w.Close() + if err1 != nil { + return err1 + } + return err2 +} + +// stoppableListener serves stoppableConn and tracks their lifetime to notify +// when it is safe to terminate the application. +type stoppableListener struct { + net.Listener + sync.WaitGroup +} + +type stoppableConn struct { + net.Conn + wg *sync.WaitGroup +} + +func newStoppableListener(l net.Listener) *stoppableListener { + return &stoppableListener{l, sync.WaitGroup{}} +} + +func (sl *stoppableListener) Accept() (net.Conn, error) { + c, err := sl.Listener.Accept() + if err != nil { + return c, err + } + sl.Add(1) + return &stoppableConn{c, &sl.WaitGroup}, nil +} + +func (sc *stoppableConn) Close() error { + sc.wg.Done() + return sc.Conn.Close() +} + +func main() { + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("l", ":8080", "on which address should the proxy listen") + flag.Parse() + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = *verbose + if err := os.MkdirAll("db", 0755); err != nil { + log.Fatal("Can't create dir", err) + } + logger, err := NewLogger("db") + if err != nil { + log.Fatal("can't open log file", err) + } + tr := transport.Transport{Proxy: transport.ProxyFromEnvironment} + // For every incoming request, override the RoundTripper to extract + // connection information. Store it is session context log it after + // handling the response. + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + ctx.RoundTripper = goproxy.RoundTripperFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (resp *http.Response, err error) { + ctx.UserData, resp, err = tr.DetailedRoundTrip(req) + return + }) + logger.LogReq(req, ctx) + return req, nil + }) + proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + logger.LogResp(resp, ctx) + return resp + }) + l, err := net.Listen("tcp", *addr) + if err != nil { + log.Fatal("listen:", err) + } + sl := newStoppableListener(l) + ch := make(chan os.Signal) + signal.Notify(ch, os.Interrupt) + go func() { + <-ch + log.Println("Got SIGINT exiting") + sl.Add(1) + sl.Close() + logger.Close() + sl.Done() + }() + log.Println("Starting Proxy") + http.Serve(sl, proxy) + sl.Wait() + log.Println("All connections closed - exit") +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/README.md b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/README.md new file mode 100644 index 000000000000..6efba22adc52 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/README.md @@ -0,0 +1,31 @@ +# Content Analysis + +`goproxy-jquery-version` starts an HTTP proxy on :8080. It checks HTML +responses, looks for scripts referencing jQuery library and emits warnings if +different versions of the library are being used for a given host. + +Start it in one shell: + +```sh +goproxy-jquery-version +``` + +Fetch goproxy homepage in another: + +```sh +http_proxy=http://127.0.0.1:8080 wget -O - \ + http://ripper234.com/p/introducing-goproxy-light-http-proxy/ +``` + +Goproxy homepage uses jQuery and a mix of plugins. First the proxy reports the +first use of jQuery it detects for the domain. Then, because the regular +expression matching the jQuery sources is imprecise, it reports a mismatch with +a plugin reference: + +```sh +2015/04/11 11:23:02 [001] WARN: ripper234.com uses //ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js +2015/04/11 11:23:02 [001] WARN: In http://ripper234.com/p/introducing-goproxy-light-http-proxy/, \ + Contradicting jqueries //ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js \ + http://ripper234.wpengine.netdna-cdn.com/wp-content/plugins/wp-ajax-edit-comments/js/jquery.colorbox.min.js?ver=5.0.36 +``` + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery1.html b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery1.html new file mode 100644 index 000000000000..26771ce34d0b --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery1.html @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery2.html b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery2.html new file mode 100644 index 000000000000..7dce036146a1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery2.html @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_homepage.html b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_homepage.html new file mode 100644 index 000000000000..27dd0b38a778 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_homepage.html @@ -0,0 +1,233 @@ + + + + + jQuery: The Write Less, Do More, JavaScript Library + + + + + + + + + +
+
+ + +
+ +
+ + + + + +
+ +
+ +
+

jQuery is a new kind of JavaScript Library.

+

jQuery is a fast and concise JavaScript Library that simplifies HTML document traversing, event handling, animating, and Ajax interactions for rapid web development. jQuery is designed to change the way that you write JavaScript.

+ +
+ +
+

Grab the latest version!

+
+
+ Choose your compression level: +
+ + jquery-1.7.2.min.js + + + jquery-1.7.2.js + +
+ +

Current Release: v1.7.2

+
+
+ +
+ + + + +
+ +
+

Learn jQuery Now!

+

What does jQuery code look like? Here's the quick and dirty:

+
+ 
$("p.neat").addClass("ohmy").show("slow");
+ Run Code + +

Congratulations! You just ran a snippet of jQuery code. Wasn't that easy? There's lots of example code throughout the documentation on this site. Be sure to give all the code a test run to see what happens.

+
+
+ + + +
+

jQuery Resources

+ +
+

Getting Started With jQuery

+ +
+
+

Developer Resources

+ +
+
+ +
+ +
+

Books About jQuery

+ + + +
+ + + +
+ + + +
+ + + + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_test.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_test.go new file mode 100644 index 000000000000..af300aaf31ad --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/jquery_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "bytes" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func equal(u, v []string) bool { + if len(u) != len(v) { + return false + } + for i, _ := range u { + if u[i] != v[i] { + return false + } + } + return true +} + +func readFile(fname string, t *testing.T) string { + b, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatal("readFile", err) + } + return string(b) +} + +func TestDefectiveScriptParser(t *testing.T) { + if l := len(findScriptSrc(` + + + + + + + `)); l != 0 { + t.Fail() + } + urls := findScriptSrc(readFile("w3schools.html", t)) + if !equal(urls, []string{"http://partner.googleadservices.com/gampad/google_service.js", + "//translate.google.com/translate_a/element.js?cb=googleTranslateElementInit"}) { + t.Error("w3schools.html", "src scripts are not recognized", urls) + } + urls = findScriptSrc(readFile("jquery_homepage.html", t)) + if !equal(urls, []string{"http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js", + "http://code.jquery.com/jquery-1.4.2.min.js", + "http://static.jquery.com/files/rocker/scripts/custom.js", + "http://static.jquery.com/donate/donate.js"}) { + t.Error("jquery_homepage.html", "src scripts are not recognized", urls) + } +} + +func proxyWithLog() (*http.Client, *bytes.Buffer) { + proxy := NewJqueryVersionProxy() + proxyServer := httptest.NewServer(proxy) + buf := new(bytes.Buffer) + proxy.Logger = log.New(buf, "", 0) + proxyUrl, _ := url.Parse(proxyServer.URL) + tr := &http.Transport{Proxy: http.ProxyURL(proxyUrl)} + client := &http.Client{Transport: tr} + return client, buf +} + +func get(t *testing.T, server *httptest.Server, client *http.Client, url string) { + resp, err := client.Get(server.URL + url) + if err != nil { + t.Fatal("cannot get proxy", err) + } + ioutil.ReadAll(resp.Body) + resp.Body.Close() +} + +func TestProxyServiceTwoVersions(t *testing.T) { + var fs = httptest.NewServer(http.FileServer(http.Dir("."))) + defer fs.Close() + + client, buf := proxyWithLog() + + get(t, fs, client, "/w3schools.html") + get(t, fs, client, "/php_man.html") + if buf.String() != "" && + !strings.Contains(buf.String(), " uses jquery ") { + t.Error("shouldn't warn on a single URL", buf.String()) + } + get(t, fs, client, "/jquery1.html") + warnings := buf.String() + if !strings.Contains(warnings, "http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js") || + !strings.Contains(warnings, "jquery.1.4.js") || + !strings.Contains(warnings, "Contradicting") { + t.Error("contradicting jquery versions (php_man.html, w3schools.html) does not issue warning", warnings) + } +} + +func TestProxyService(t *testing.T) { + var fs = httptest.NewServer(http.FileServer(http.Dir("."))) + defer fs.Close() + + client, buf := proxyWithLog() + + get(t, fs, client, "/jquery_homepage.html") + warnings := buf.String() + if !strings.Contains(warnings, "http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js") || + !strings.Contains(warnings, "http://code.jquery.com/jquery-1.4.2.min.js") || + !strings.Contains(warnings, "Contradicting") { + t.Error("contradicting jquery versions does not issue warning") + } +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/main.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/main.go new file mode 100644 index 000000000000..a92dddeacb21 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/html" + "log" + "net/http" + "regexp" +) + +var ( + // who said we can't parse HTML with regexp? + scriptMatcher = regexp.MustCompile(`(?i:]*\ssrc=["']([^"']*)["'])`) +) + +// findScripts returns all sources of HTML script tags found in input text. +func findScriptSrc(html string) []string { + srcs := make([]string, 0) + matches := scriptMatcher.FindAllStringIndex(html, -1) + for _, match := range matches { + // -1 to capture the whitespace at the end of the script tag + srcMatch := srcAttrMatcher.FindStringSubmatch(html[match[1]-1:]) + if srcMatch != nil { + srcs = append(srcs, srcMatch[1]) + } + } + return srcs +} + +// NewJQueryVersionProxy creates a proxy checking responses HTML content, looks +// for scripts referencing jQuery library and emits warnings if different +// versions of the library are being used for a given host. +func NewJqueryVersionProxy() *goproxy.ProxyHttpServer { + proxy := goproxy.NewProxyHttpServer() + m := make(map[string]string) + jqueryMatcher := regexp.MustCompile(`(?i:jquery\.)`) + proxy.OnResponse(goproxy_html.IsHtml).Do(goproxy_html.HandleString( + func(s string, ctx *goproxy.ProxyCtx) string { + for _, src := range findScriptSrc(s) { + if !jqueryMatcher.MatchString(src) { + continue + } + prev, ok := m[ctx.Req.Host] + if ok { + if prev != src { + ctx.Warnf("In %v, Contradicting jqueries %v %v", + ctx.Req.URL, prev, src) + break + } + } else { + ctx.Warnf("%s uses jquery %s", ctx.Req.Host, src) + m[ctx.Req.Host] = src + } + } + return s + })) + return proxy +} + +func main() { + proxy := NewJqueryVersionProxy() + log.Fatal(http.ListenAndServe(":8080", proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/php_man.html b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/php_man.html new file mode 100644 index 000000000000..1159d762d6ae --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/php_man.html @@ -0,0 +1,323 @@ + + + + PHP: PHP Manual - Manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+

+ + search for + + + in the + + + +

+
+
+ +
+
+ + + +
+
+ +
+ + + +   + +
+ [edit] Last updated: Fri, 23 Mar 2012 +
+
+

view this page in

+ +
+
+
+ + +
+

PHP Manual

+ + + +
+
+ +
by:
+ + Mehdi Achour + +
+ + +
+ + Friedhelm Betz + +
+ + +
+ + Antony Dovgal + +
+ + +
+ + Nuno Lopes + +
+ + +
+ + Hannes Magnusson + +
+ + +
+ + Georg Richter + +
+ + +
+ + Damien Seguy + +
+ + +
+ + Jakub Vrana + +
+ + + +
+ + + And several others + + +
+ +
+
2012-03-23
+ +
+
Edited By: + + Philip Olson + +
+ +
+ +
© + 1997-2012 + the PHP Documentation Group +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +


+
+
+ add a note add a note + User Contributed Notes + PHP Manual +
+
There are no user contributed notes for this page.

+
+
 
+
+ + + + + + + \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/w3schools.html b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/w3schools.html new file mode 100644 index 000000000000..ecf3a9dfec37 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-jquery-version/w3schools.html @@ -0,0 +1,1610 @@ + + + + + + + + +HTML5 Tutorial + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ +
+ +
+ + W3Schools.com + +
+ + + +
+ + + + + +
+ + + + + +
+ +
+ +
+ +
+ +
+ + HOME + + HTML + + CSS + + XML + + JAVASCRIPT + + ASP + + PHP + + SQL + + MORE... + +
+ +
+ + REFERENCES | + + EXAMPLES | + + FORUM | + + ABOUT + +
+ +
+ + + +
+ + + + + + + + + + + + + +
+ + + + + + + + + +
+ +
+ +

HTML5 Tutorial

+ +HTML5 Home
+ +HTML5 Introduction
+ +HTML5 New Elements
+ +HTML5 Video
+ +HTML5 Video/DOM
+ +HTML5 Audio
+ +HTML5 Drag and Drop
+ +HTML5 Canvas
+ +HTML5 SVG
+ +HTML5 Canvas vs. SVG
+ +HTML5 Geolocation
+ +HTML5 Web Storage
+ +HTML5 App Cache
+ +HTML5 Web Workers
+ +HTML5 SSE
+ +
+ +

HTML5 Forms

+ +HTML5 Input Types
+ +HTML5 Form Elements
+ +HTML5 Form Attributes
+ +
+ +

HTML5 Reference

+ +HTML5 Tags
+ +HTML5 Attributes
+ +HTML5 Events
+ +HTML5 Audio/Video
+ +HTML5 Canvas 2d
+ +HTML Valid DTDs
+ +
+ +

HTML5 Tags

+ +<!-->
+ +<!DOCTYPE>
+ +<a>
+ +<abbr>
+ +<acronym>
+ +<address>
+ +<applet>
+ +<area>
+ +<article>
+ +<aside>
+ +<audio>
+ +<b>
+ +<base>
+ +<basefont>
+ +<bdi>
+ +<bdo>
+ +<big>
+ +<blockquote>
+ +<body>
+ +<br>
+ +<button>
+ +<canvas>
+ +<caption>
+ +<center>
+ +<cite>
+ +<code>
+ +<col>
+ +<colgroup>
+ +<command>
+ +<datalist>
+ +<dd>
+ +<del>
+ +<details>
+ +<dfn>
+ +<dir>
+ +<div>
+ +<dl>
+ +<dt>
+ +<em>
+ +<embed>
+ +<fieldset>
+ +<figcaption>
+ +<figure>
+ +<font>
+ +<footer>
+ +<form>
+ +<frame>
+ +<frameset>
+ +<h1> - <h6>
+ +<head>
+ +<header>
+ +<hgroup>
+ +<hr>
+ +<html>
+ +<i>
+ +<iframe>
+ +<img>
+ +<input>
+ +<ins>
+ +<keygen>
+ +<kbd>
+ +<label>
+ +<legend>
+ +<li>
+ +<link>
+ +<map>
+ +<mark>
+ +<menu>
+ +<meta>
+ +<meter>
+ +<nav>
+ +<noframes>
+ +<noscript>
+ +<object>
+ +<ol>
+ +<optgroup>
+ +<option>
+ +<output>
+ +<p>
+ +<param>
+ +<pre>
+ +<progress>
+ +<q>
+ +<rp>
+ +<rt>
+ +<ruby>
+ +<s>
+ +<samp>
+ +<script>
+ +<section>
+ +<select>
+ +<small>
+ +<source>
+ +<span>
+ +<strike>
+ +<strong>
+ +<style>
+ +<sub>
+ +<summary>
+ +<sup>
+ +<table>
+ +<tbody>
+ +<td>
+ +<textarea>
+ +<tfoot>
+ +<th>
+ +<thead>
+ +<time>
+ +<title>
+ +<tr>
+ +<track>
+ +<tt>
+ +<u>
+ +<ul>
+ +<var>
+ +<video>
+ +<wbr>
+ +
+ +
+ + + +

HTML5 Tutorial

+ +
+ + + + + +
+ +
+ +
+ +
HTML5 is The New HTML Standard
+ +
+ + + + + +
+ + + + + + + + + + + +
+ +
+ +
+ +

HTML5

+ +
    + +
  • New Elements
    • + +
  • New Attributes
    • + +
  • Full CSS3 Support
    • + +
  • Video and Audio
    • + +
  • 2D/3D Graphics
    • + +
  • Local Storage
    • + +
  • Local SQL Database
    • + +
  • Web Applications
    • + +
+ +
+ + + + + + + + + + + + + + + + + + + +
+ +
+ +
    + +
  • + +
  • + +
  • + +
  • + +
  • + +
  • + +
+ +
+ +
+ +
+ +

Examples in Each Chapter

+ +

With our HTML editor, you can edit the HTML, and click on a button to view the result.

+ +
+ +

Example

+ +
+ + <!DOCTYPE HTML>
+ + <html>
+ + <body>
+ +
+ + <video width="320" height="240" controls="controls">
+ +  <source src="movie.mp4" type="video/mp4" />
+ +  <source src="movie.ogg" type="video/ogg" />
+ +  <source src="movie.webm" type="video/webm" />
+ + Your browser does not support the video tag.
+ + </video>
+ +
+ + </body>
+ + </html> + +
+ +
+ + Try it yourself »
+ +

Click on the "Try it yourself" button to see how it works

+ +

Start learning HTML5 now!

+ + + +

HTML5 References

+ +

At W3Schools you will find complete references about tags, global attributes, + +standard events, and more.

+ +

+ +HTML5 Tag Reference + +

+ + + +
+ +
+ + + + + +
+ + + + + + + + + + + + + + + + + + + +
+ + + +
+ + + + + + + + + + + + + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + +
WEB HOSTING
+ +Best Web Hosting + +
+ +PHP MySQL Hosting + +
+ +Best Hosting Coupons + +
+ +UK Reseller Hosting + +
+ +Cloud Hosting + +
+ +Top Web Hosting + +
+ +$3.98 Unlimited Hosting + +
+ +Premium Website Design + +
+ + + + + + + + + + + + + +
WEB BUILDING
+ + + +Download XML Editor + + + +
+ +FREE Website BUILDER + +
+ +Free Website Templates + +Free CSS Templates + +
+ +CREATE HTML Websites + +
+ + + + + + + +
W3SCHOOLS EXAMS
+ +Get Certified in:
HTML, CSS, JavaScript, XML, PHP, and ASP + +
+ + + + + + + +
W3SCHOOLS BOOKS
+ + + +New Books:
HTML, CSS
+ +JavaScript, and Ajax + +
+ + + + + +
STATISTICS
+ +Browser Statistics
+ +Browser OS
+ +Browser Display + +
+ + + + + + + + + + + + + +
SHARE THIS PAGE
+ + + +
+ + + + + + + +

+ +
+ + + + + +
+ +
+ + + +
+ +
+ +
+ +
+ +
+ + + +
+ +
+ + + + + + + + + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/README.md b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/README.md new file mode 100644 index 000000000000..23b52240533a --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/README.md @@ -0,0 +1,21 @@ +# Request Filtering + +`goproxy-no-reddit-at-work` starts an HTTP proxy on :8080. It denies requests +to "www.reddit.com" made between 8am to 5pm inclusive, local time. + +Start it in one shell: + +```sh +$ goproxy-no-reddit-at-work +``` + +Fetch reddit in another: + +```sh +$ http_proxy=http://127.0.0.1:8080 wget -O - http://www.reddit.com +--2015-04-11 16:59:01-- http://www.reddit.com/ +Connecting to 127.0.0.1:8080... connected. +Proxy request sent, awaiting response... 403 Forbidden +2015-04-11 16:59:01 ERROR 403: Forbidden. +``` + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/noreddit.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/noreddit.go new file mode 100644 index 000000000000..b174845929c3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-no-reddit-at-worktime/noreddit.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/elazarl/goproxy" + "log" + "net/http" + "time" +) + +func main() { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest(goproxy.DstHostIs("www.reddit.com")).DoFunc( + func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + h, _, _ := time.Now().Clock() + if h >= 8 && h <= 17 { + return r, goproxy.NewResponse(r, + goproxy.ContentTypeText, http.StatusForbidden, + "Don't waste your time!") + } else { + ctx.Warnf("clock: %d, you can waste your time...", h) + } + return r, nil + }) + log.Fatalln(http.ListenAndServe(":8080", proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sokeepalive/sokeepalive.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sokeepalive/sokeepalive.go new file mode 100644 index 000000000000..86d8bcfcb5c7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sokeepalive/sokeepalive.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/elazarl/goproxy" + "log" + "flag" + "net" + "net/http" +) + +func main() { + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + flag.Parse() + proxy := goproxy.NewProxyHttpServer() + proxy.Tr.Dial = func(network, addr string) (c net.Conn, err error) { + c, err = net.Dial(network, addr) + if c, ok := c.(*net.TCPConn); err != nil && ok { + c.SetKeepAlive(true) + } + return + } + proxy.Verbose = *verbose + log.Fatal(http.ListenAndServe(*addr, proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sslstrip/sslstrip.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sslstrip/sslstrip.go new file mode 100644 index 000000000000..b7e2dc9e5ec1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-sslstrip/sslstrip.go @@ -0,0 +1,24 @@ +package main + +import ( + "github.com/elazarl/goproxy" + "log" + "flag" + "net/http" +) + +func main() { + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + flag.Parse() + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) + proxy.OnRequest().DoFunc(func (req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + if req.URL.Scheme == "https" { + req.URL.Scheme = "http" + } + return req, nil + }) + proxy.Verbose = *verbose + log.Fatal(http.ListenAndServe(*addr, proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/README.md b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/README.md new file mode 100644 index 000000000000..a51d4c8ebcfb --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/README.md @@ -0,0 +1,43 @@ +# Gather Browsing Statistics + +`goproxy-stats` starts an HTTP proxy on :8080, counts the bytes received for +web resources and prints the cumulative sum per URL every 20 seconds. + +Start it in one shell: + +```sh +goproxy-stats +``` + +Fetch goproxy homepage in another: + +```sh +mkdir tmp +cd tmp +http_proxy=http://127.0.0.1:8080 wget -r -l 1 -H \ + http://ripper234.com/p/introducing-goproxy-light-http-proxy/ +``` + +Stop it after a moment. `goproxy-stats` should eventually print: +```sh +listening on :8080 +statistics +http://www.telerik.com/fiddler -> 84335 +http://msmvps.com/robots.txt -> 157 +http://eli.thegreenplace.net/robots.txt -> 294 +http://www.phdcomics.com/robots.txt -> 211 +http://resharper.blogspot.com/robots.txt -> 221 +http://idanz.blogli.co.il/robots.txt -> 271 +http://ripper234.com/p/introducing-goproxy-light-http-proxy/ -> 44407 +http://live.gnome.org/robots.txt -> 298 +http://ponetium.wordpress.com/robots.txt -> 178 +http://pilaheleg.blogli.co.il/robots.txt -> 321 +http://pilaheleg.wordpress.com/robots.txt -> 178 +http://blogli.co.il/ -> 9165 +http://nimrod-code.org/robots.txt -> 289 +http://www.joelonsoftware.com/robots.txt -> 1245 +http://top-performance.blogspot.com/robots.txt -> 227 +http://ooc-lang.org/robots.txt -> 345 +http://blogs.jetbrains.com/robots.txt -> 293 +``` + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/main.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/main.go new file mode 100644 index 000000000000..e4cde8d9399e --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-stats/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/html" + "io" + "log" + . "net/http" + "time" +) + +type Count struct { + Id string + Count int64 +} +type CountReadCloser struct { + Id string + R io.ReadCloser + ch chan<- Count + nr int64 +} + +func (c *CountReadCloser) Read(b []byte) (n int, err error) { + n, err = c.R.Read(b) + c.nr += int64(n) + return +} +func (c CountReadCloser) Close() error { + c.ch <- Count{c.Id, c.nr} + return c.R.Close() +} + +func main() { + proxy := goproxy.NewProxyHttpServer() + timer := make(chan bool) + ch := make(chan Count, 10) + go func() { + for { + time.Sleep(20 * time.Second) + timer <- true + } + }() + go func() { + m := make(map[string]int64) + for { + select { + case c := <-ch: + m[c.Id] = m[c.Id] + c.Count + case <-timer: + fmt.Printf("statistics\n") + for k, v := range m { + fmt.Printf("%s -> %d\n", k, v) + } + } + } + }() + + // IsWebRelatedText filters on html/javascript/css resources + proxy.OnResponse(goproxy_html.IsWebRelatedText).DoFunc(func(resp *Response, ctx *goproxy.ProxyCtx) *Response { + resp.Body = &CountReadCloser{ctx.Req.URL.String(), resp.Body, ch, 0} + return resp + }) + fmt.Printf("listening on :8080\n") + log.Fatal(ListenAndServe(":8080", proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/README.md b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/README.md new file mode 100644 index 000000000000..7edb0989bf3d --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/README.md @@ -0,0 +1,17 @@ +# Transparent Proxy + +This transparent example in goproxy is meant to show how to transparenty proxy and hijack all http and https connections while doing a man-in-the-middle to the TLS session. It requires that goproxy sees all the packets traversing out to the internet. Linux iptables rules deal with changing the source/destination IPs to act transparently, but you do need to setup your network configuration so that goproxy is a mandatory stop on the outgoing route. Primarily you can do this by placing the proxy inline. goproxy does not have any WCCP support itself; patches welcome. + +## Why not explicit? + +Transparent proxies are more difficult to maintain and setup from a server side, but they require no configuration on the client(s) which could be in unmanaged systems or systems that don't support a proxy configuration. See the [eavesdropper example](https://github.com/elazarl/goproxy/blob/master/examples/goproxy-eavesdropper/main.go) if you want to see an explicit proxy example. + +## Potential Issues + +Support for very old clients using HTTPS will fail. Clients need to send the SNI value in the TLS ClientHello which most modern clients do these days, but old clients will break. + +If you're routing table allows for it, an explicit http request to goproxy will cause it to fail in an endless loop since it will try to request resources from itself repeatedly. This could be solved in the goproxy code by looking up the hostnames, but it adds a delay that is much easier/faster to handle on the routing side. + +## Routing Rules + +Example routing rules are included in [proxy.sh](https://github.com/elazarl/goproxy/blob/master/examples/goproxy-transparent/proxy.sh) but are best when setup using your distribution's configuration. diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/proxy.sh b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/proxy.sh new file mode 100644 index 000000000000..c63111432e2f --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/proxy.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# goproxy IP +GOPROXY_SERVER="10.10.10.1" +# goproxy port +GOPROXY_PORT="3129" +GOPROXY_PORT_TLS="3128" +# DO NOT MODIFY BELOW +# Load IPTABLES modules for NAT and IP conntrack support +modprobe ip_conntrack +modprobe ip_conntrack_ftp +echo 1 > /proc/sys/net/ipv4/ip_forward +echo 2 > /proc/sys/net/ipv4/conf/all/rp_filter + +# Clean old firewall +iptables -t nat -F +iptables -t nat -X +iptables -t mangle -F +iptables -t mangle -X + +# Write new rules +iptables -t nat -A PREROUTING -s $GOPROXY_SERVER -p tcp --dport $GOPROXY_PORT -j ACCEPT +iptables -t nat -A PREROUTING -s $GOPROXY_SERVER -p tcp --dport $GOPROXY_PORT_TLS -j ACCEPT +iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination $GOPROXY_SERVER:$GOPROXY_PORT +iptables -t nat -A PREROUTING -p tcp --dport 443 -j DNAT --to-destination $GOPROXY_SERVER:$GOPROXY_PORT_TLS +# The following line supports using goproxy as an explicit proxy in addition +iptables -t nat -A PREROUTING -p tcp --dport 8080 -j DNAT --to-destination $GOPROXY_SERVER:$GOPROXY_PORT +iptables -t nat -A POSTROUTING -j MASQUERADE +iptables -t mangle -A PREROUTING -p tcp --dport $GOPROXY_PORT -j DROP +iptables -t mangle -A PREROUTING -p tcp --dport $GOPROXY_PORT_TLS -j DROP diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/transparent.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/transparent.go new file mode 100644 index 000000000000..b4134e23b720 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-transparent/transparent.go @@ -0,0 +1,148 @@ +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "log" + "net" + "net/http" + "net/url" + "regexp" + + "github.com/elazarl/goproxy" + "github.com/inconshreveable/go-vhost" +) + +func orPanic(err error) { + if err != nil { + panic(err) + } +} + +func main() { + verbose := flag.Bool("v", true, "should every proxy request be logged to stdout") + http_addr := flag.String("httpaddr", ":3129", "proxy http listen address") + https_addr := flag.String("httpsaddr", ":3128", "proxy https listen address") + flag.Parse() + + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = *verbose + if proxy.Verbose { + log.Printf("Server starting up! - configured to listen on http interface %s and https interface %s", *http_addr, *https_addr) + } + + proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host == "" { + fmt.Fprintln(w, "Cannot handle requests without Host header, e.g., HTTP 1.0") + return + } + req.URL.Scheme = "http" + req.URL.Host = req.Host + proxy.ServeHTTP(w, req) + }) + proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))). + HandleConnect(goproxy.AlwaysMitm) + proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*:80$"))). + HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) { + defer func() { + if e := recover(); e != nil { + ctx.Logf("error connecting to remote: %v", e) + client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n")) + } + client.Close() + }() + clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client)) + remote, err := connectDial(proxy, "tcp", req.URL.Host) + orPanic(err) + remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote)) + for { + req, err := http.ReadRequest(clientBuf.Reader) + orPanic(err) + orPanic(req.Write(remoteBuf)) + orPanic(remoteBuf.Flush()) + resp, err := http.ReadResponse(remoteBuf.Reader, req) + orPanic(err) + orPanic(resp.Write(clientBuf.Writer)) + orPanic(clientBuf.Flush()) + } + }) + + go func() { + log.Fatalln(http.ListenAndServe(*http_addr, proxy)) + }() + + // listen to the TLS ClientHello but make it a CONNECT request instead + ln, err := net.Listen("tcp", *https_addr) + if err != nil { + log.Fatalf("Error listening for https connections - %v", err) + } + for { + c, err := ln.Accept() + if err != nil { + log.Printf("Error accepting new connection - %v", err) + continue + } + go func(c net.Conn) { + tlsConn, err := vhost.TLS(c) + if err != nil { + log.Printf("Error accepting new connection - %v", err) + } + if tlsConn.Host() == "" { + log.Printf("Cannot support non-SNI enabled clients") + return + } + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{ + Opaque: tlsConn.Host(), + Host: net.JoinHostPort(tlsConn.Host(), "443"), + }, + Host: tlsConn.Host(), + Header: make(http.Header), + } + resp := dumbResponseWriter{tlsConn} + proxy.ServeHTTP(resp, connectReq) + }(c) + } +} + +// copied/converted from https.go +func dial(proxy *goproxy.ProxyHttpServer, network, addr string) (c net.Conn, err error) { + if proxy.Tr.Dial != nil { + return proxy.Tr.Dial(network, addr) + } + return net.Dial(network, addr) +} + +// copied/converted from https.go +func connectDial(proxy *goproxy.ProxyHttpServer, network, addr string) (c net.Conn, err error) { + if proxy.ConnectDial == nil { + return dial(proxy, network, addr) + } + return proxy.ConnectDial(network, addr) +} + +type dumbResponseWriter struct { + net.Conn +} + +func (dumb dumbResponseWriter) Header() http.Header { + panic("Header() should not be called on this ResponseWriter") +} + +func (dumb dumbResponseWriter) Write(buf []byte) (int, error) { + if bytes.Equal(buf, []byte("HTTP/1.0 200 OK\r\n\r\n")) { + return len(buf), nil // throw away the HTTP OK response from the faux CONNECT request + } + return dumb.Conn.Write(buf) +} + +func (dumb dumbResponseWriter) WriteHeader(code int) { + panic("WriteHeader() should not be called on this ResponseWriter") +} + +func (dumb dumbResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return dumb, bufio.NewReadWriter(bufio.NewReader(dumb), bufio.NewWriter(dumb)), nil +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-upside-down-ternet/main.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-upside-down-ternet/main.go new file mode 100644 index 000000000000..4b683fd329a0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-upside-down-ternet/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/image" + "image" + "log" + "net/http" +) + +func main() { + proxy := goproxy.NewProxyHttpServer() + proxy.OnResponse().Do(goproxy_image.HandleImage(func(img image.Image, ctx *goproxy.ProxyCtx) image.Image { + dx, dy := img.Bounds().Dx(), img.Bounds().Dy() + + nimg := image.NewRGBA(img.Bounds()) + for i := 0; i < dx; i++ { + for j := 0; j <= dy; j++ { + nimg.Set(i, j, img.At(i, dy-j-1)) + } + } + return nimg + })) + proxy.Verbose = true + log.Fatal(http.ListenAndServe(":8080", proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-yui-minify/yui.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-yui-minify/yui.go new file mode 100644 index 000000000000..0e7eadbb157c --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/examples/goproxy-yui-minify/yui.go @@ -0,0 +1,91 @@ +// This example would minify standalone Javascript files (identified by their content type) +// using the command line utility YUI compressor http://yui.github.io/yuicompressor/ +// Example usage: +// +// ./yui -java /usr/local/bin/java -yuicompressor ~/Downloads/yuicompressor-2.4.8.jar +// $ curl -vx localhost:8080 http://golang.org/lib/godoc/godocs.js +// (function(){function g(){var u=$("#search");if(u.length===0){return}function t(){if(.... +// $ curl http://golang.org/lib/godoc/godocs.js | head -n 3 +// // Copyright 2012 The Go Authors. All rights reserved. +// // Use of this source code is governed by a BSD-style +// // license that can be found in the LICENSE file. +package main + +import ( + "flag" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "strings" + + "github.com/elazarl/goproxy" +) + +func main() { + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + java := flag.String("javapath", "java", "where the Java executable is located") + yuicompressor := flag.String("yuicompressor", "", "where the yuicompressor is located, assumed to be in CWD") + yuicompressordir := flag.String("yuicompressordir", ".", "a folder to search yuicompressor in, will be ignored if yuicompressor is set") + flag.Parse() + if *yuicompressor == "" { + files, err := ioutil.ReadDir(*yuicompressordir) + if err != nil { + log.Fatal("Cannot find yuicompressor jar") + } + for _, file := range files { + if strings.HasPrefix(file.Name(), "yuicompressor") && strings.HasSuffix(file.Name(), ".jar") { + c := path.Join(*yuicompressordir, file.Name()) + yuicompressor = &c + break + } + } + } + if *yuicompressor == "" { + log.Fatal("Can't find yuicompressor jar, searched yuicompressor*.jar in dir ", *yuicompressordir) + } + if _, err := os.Stat(*yuicompressor); os.IsNotExist(err) { + log.Fatal("Can't find yuicompressor jar specified ", *yuicompressor) + } + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = *verbose + proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + contentType := resp.Header.Get("Content-Type") + if contentType == "application/javascript" || contentType == "application/x-javascript" { + // in real code, response should be streamed as well + var err error + cmd := exec.Command(*java, "-jar", *yuicompressor, "--type", "js") + cmd.Stdin = resp.Body + resp.Body, err = cmd.StdoutPipe() + if err != nil { + ctx.Warnf("Cannot minify content in %v: %v", ctx.Req.URL, err) + return goproxy.TextResponse(ctx.Req, "Error getting stdout pipe") + } + stderr, err := cmd.StderrPipe() + if err != nil { + ctx.Logf("Error obtaining stderr from yuicompress: %s", err) + return goproxy.TextResponse(ctx.Req, "Error getting stderr pipe") + } + if err := cmd.Start(); err != nil { + ctx.Warnf("Cannot minify content in %v: %v", ctx.Req.URL, err) + } + go func() { + defer stderr.Close() + const kb = 1024 + msg, err := ioutil.ReadAll(&io.LimitedReader{stderr, 50 * kb}) + if len(msg) != 0 { + ctx.Logf("Error executing yuicompress: %s", string(msg)) + } + if err != nil { + ctx.Logf("Error reading stderr from yuicompress: %s", string(msg)) + } + }() + } + return resp + }) + log.Fatal(http.ListenAndServe(*addr, proxy)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic.go new file mode 100644 index 000000000000..4833763e2a81 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic.go @@ -0,0 +1,76 @@ +package auth + +import ( + "bytes" + "encoding/base64" + "io/ioutil" + "net/http" + "strings" + + "github.com/elazarl/goproxy" +) + +var unauthorizedMsg = []byte("407 Proxy Authentication Required") + +func BasicUnauthorized(req *http.Request, realm string) *http.Response { + // TODO(elazar): verify realm is well formed + return &http.Response{ + StatusCode: 407, + ProtoMajor: 1, + ProtoMinor: 1, + Request: req, + Header: http.Header{"Proxy-Authenticate": []string{"Basic realm=" + realm}}, + Body: ioutil.NopCloser(bytes.NewBuffer(unauthorizedMsg)), + ContentLength: int64(len(unauthorizedMsg)), + } +} + +var proxyAuthorizatonHeader = "Proxy-Authorization" + +func auth(req *http.Request, f func(user, passwd string) bool) bool { + authheader := strings.SplitN(req.Header.Get(proxyAuthorizatonHeader), " ", 2) + req.Header.Del(proxyAuthorizatonHeader) + if len(authheader) != 2 || authheader[0] != "Basic" { + return false + } + userpassraw, err := base64.StdEncoding.DecodeString(authheader[1]) + if err != nil { + return false + } + userpass := strings.SplitN(string(userpassraw), ":", 2) + if len(userpass) != 2 { + return false + } + return f(userpass[0], userpass[1]) +} + +// Basic returns a basic HTTP authentication handler for requests +// +// You probably want to use auth.ProxyBasic(proxy) to enable authentication for all proxy activities +func Basic(realm string, f func(user, passwd string) bool) goproxy.ReqHandler { + return goproxy.FuncReqHandler(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + if !auth(req, f) { + return nil, BasicUnauthorized(req, realm) + } + return req, nil + }) +} + +// BasicConnect returns a basic HTTP authentication handler for CONNECT requests +// +// You probably want to use auth.ProxyBasic(proxy) to enable authentication for all proxy activities +func BasicConnect(realm string, f func(user, passwd string) bool) goproxy.HttpsHandler { + return goproxy.FuncHttpsHandler(func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + if !auth(ctx.Req, f) { + ctx.Resp = BasicUnauthorized(ctx.Req, realm) + return goproxy.RejectConnect, host + } + return goproxy.OkConnect, host + }) +} + +// ProxyBasic will force HTTP authentication before any request to the proxy is processed +func ProxyBasic(proxy *goproxy.ProxyHttpServer, realm string, f func(user, passwd string) bool) { + proxy.OnRequest().Do(Basic(realm, f)) + proxy.OnRequest().HandleConnect(BasicConnect(realm, f)) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic_test.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic_test.go new file mode 100644 index 000000000000..792d789bcd26 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/auth/basic_test.go @@ -0,0 +1,175 @@ +package auth_test + +import ( + "encoding/base64" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "os/signal" + "sync/atomic" + "testing" + + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/auth" +) + +type ConstantHanlder string + +func (h ConstantHanlder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, string(h)) +} + +func oneShotProxy(proxy *goproxy.ProxyHttpServer) (client *http.Client, s *httptest.Server) { + s = httptest.NewServer(proxy) + + proxyUrl, _ := url.Parse(s.URL) + tr := &http.Transport{Proxy: http.ProxyURL(proxyUrl)} + client = &http.Client{Transport: tr} + return +} + +func times(n int, s string) string { + r := make([]byte, 0, n*len(s)) + for i := 0; i < n; i++ { + r = append(r, s...) + } + return string(r) +} + +func TestBasicConnectAuthWithCurl(t *testing.T) { + expected := ":c>" + background := httptest.NewTLSServer(ConstantHanlder(expected)) + defer background.Close() + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().HandleConnect(auth.BasicConnect("my_realm", func(user, passwd string) bool { + return user == "user" && passwd == "open sesame" + })) + _, proxyserver := oneShotProxy(proxy) + defer proxyserver.Close() + + cmd := exec.Command("curl", + "--silent", "--show-error", "--insecure", + "-x", proxyserver.URL, + "-U", "user:open sesame", + "-p", + "--url", background.URL+"/[1-3]", + ) + out, err := cmd.CombinedOutput() // if curl got error, it'll show up in stderr + if err != nil { + t.Fatal(err, string(out)) + } + finalexpected := times(3, expected) + if string(out) != finalexpected { + t.Error("Expected", finalexpected, "got", string(out)) + } +} + +func TestBasicAuthWithCurl(t *testing.T) { + expected := ":c>" + background := httptest.NewServer(ConstantHanlder(expected)) + defer background.Close() + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().Do(auth.Basic("my_realm", func(user, passwd string) bool { + return user == "user" && passwd == "open sesame" + })) + _, proxyserver := oneShotProxy(proxy) + defer proxyserver.Close() + + cmd := exec.Command("curl", + "--silent", "--show-error", + "-x", proxyserver.URL, + "-U", "user:open sesame", + "--url", background.URL+"/[1-3]", + ) + out, err := cmd.CombinedOutput() // if curl got error, it'll show up in stderr + if err != nil { + t.Fatal(err, string(out)) + } + finalexpected := times(3, expected) + if string(out) != finalexpected { + t.Error("Expected", finalexpected, "got", string(out)) + } +} + +func TestBasicAuth(t *testing.T) { + expected := "hello" + background := httptest.NewServer(ConstantHanlder(expected)) + defer background.Close() + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().Do(auth.Basic("my_realm", func(user, passwd string) bool { + return user == "user" && passwd == "open sesame" + })) + client, proxyserver := oneShotProxy(proxy) + defer proxyserver.Close() + + // without auth + resp, err := client.Get(background.URL) + if err != nil { + t.Fatal(err) + } + if resp.Header.Get("Proxy-Authenticate") != "Basic realm=my_realm" { + t.Error("Expected Proxy-Authenticate header got", resp.Header.Get("Proxy-Authenticate")) + } + if resp.StatusCode != 407 { + t.Error("Expected status 407 Proxy Authentication Required, got", resp.Status) + } + + // with auth + req, err := http.NewRequest("GET", background.URL, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Proxy-Authorization", + "Basic "+base64.StdEncoding.EncodeToString([]byte("user:open sesame"))) + resp, err = client.Do(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + t.Error("Expected status 200 OK, got", resp.Status) + } + msg, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(msg) != "hello" { + t.Errorf("Expected '%s', actual '%s'", expected, string(msg)) + } +} + +func TestWithBrowser(t *testing.T) { + // an easy way to check if auth works with webserver + // to test, run with + // $ go test -run TestWithBrowser -- server + // configure a browser to use the printed proxy address, use the proxy + // and exit with Ctrl-C. It will throw error if your haven't acutally used the proxy + if os.Args[len(os.Args)-1] != "server" { + return + } + proxy := goproxy.NewProxyHttpServer() + println("proxy localhost port 8082") + access := int32(0) + proxy.OnRequest().Do(auth.Basic("my_realm", func(user, passwd string) bool { + atomic.AddInt32(&access, 1) + return user == "user" && passwd == "1234" + })) + l, err := net.Listen("tcp", "localhost:8082") + if err != nil { + t.Fatal(err) + } + ch := make(chan os.Signal) + signal.Notify(ch, os.Interrupt) + go func() { + <-ch + l.Close() + }() + http.Serve(l, proxy) + if access <= 0 { + t.Error("No one accessed the proxy") + } +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.html b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.html new file mode 100644 index 000000000000..22c54b1629b1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.html @@ -0,0 +1,585 @@ + + + + + + + + + +ãó äáéú ùì ùä"í + + + + + + + + + + + + + + + + + +
+ + +
+ +
+ + + +
+
+ + +rss +
+ +
+ + + + +
+
+
+  àúøé 2012á ðôúçå ìñèåãðèéí. öååú ùä"í îàçì ìëí ñîñèø îåöìç. +(4.3.12) +
+
+ + + + + + +
+ + +
+
+ +
+ + +
+ + +
+ + + + + + + diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.txt b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.txt new file mode 100644 index 000000000000..ef904ced9e9c --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/cp1255.txt @@ -0,0 +1 @@ +ãó \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html.go new file mode 100644 index 000000000000..b438d373410d --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html.go @@ -0,0 +1,104 @@ +// extension to goproxy that will allow you to easily filter web browser related content. +package goproxy_html + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "net/http" + "strings" + + "code.google.com/p/go-charset/charset" + _ "code.google.com/p/go-charset/data" + "github.com/elazarl/goproxy" +) + +var IsHtml goproxy.RespCondition = goproxy.ContentTypeIs("text/html") + +var IsCss goproxy.RespCondition = goproxy.ContentTypeIs("text/css") + +var IsJavaScript goproxy.RespCondition = goproxy.ContentTypeIs("text/javascript", + "application/javascript") + +var IsJson goproxy.RespCondition = goproxy.ContentTypeIs("text/json") + +var IsXml goproxy.RespCondition = goproxy.ContentTypeIs("text/xml") + +var IsWebRelatedText goproxy.RespCondition = goproxy.ContentTypeIs("text/html", + "text/css", + "text/javascript", "application/javascript", + "text/xml", + "text/json") + +// HandleString will receive a function that filters a string, and will convert the +// request body to a utf8 string, according to the charset specified in the Content-Type +// header. +// guessing Html charset encoding from the tags is not yet implemented. +func HandleString(f func(s string, ctx *goproxy.ProxyCtx) string) goproxy.RespHandler { + return HandleStringReader(func(r io.Reader, ctx *goproxy.ProxyCtx) io.Reader { + b, err := ioutil.ReadAll(r) + if err != nil { + ctx.Warnf("Cannot read string from resp body: %v", err) + return r + } + return bytes.NewBufferString(f(string(b), ctx)) + }) +} + +// Will receive an input stream which would convert the response to utf-8 +// The given function must close the reader r, in order to close the response body. +func HandleStringReader(f func(r io.Reader, ctx *goproxy.ProxyCtx) io.Reader) goproxy.RespHandler { + return goproxy.FuncRespHandler(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + if ctx.Error != nil { + return nil + } + charsetName := ctx.Charset() + if charsetName == "" { + charsetName = "utf-8" + } + + if strings.ToLower(charsetName) != "utf-8" { + r, err := charset.NewReader(charsetName, resp.Body) + if err != nil { + ctx.Warnf("Cannot convert from %v to utf-8: %v", charsetName, err) + return resp + } + tr, err := charset.TranslatorTo(charsetName) + if err != nil { + ctx.Warnf("Can't translate to %v from utf-8: %v", charsetName, err) + return resp + } + if err != nil { + ctx.Warnf("Cannot translate to %v: %v", charsetName, err) + return resp + } + newr := charset.NewTranslatingReader(f(r, ctx), tr) + resp.Body = &readFirstCloseBoth{ioutil.NopCloser(newr), resp.Body} + } else { + //no translation is needed, already at utf-8 + resp.Body = &readFirstCloseBoth{ioutil.NopCloser(f(resp.Body, ctx)), resp.Body} + } + return resp + }) +} + +type readFirstCloseBoth struct { + r io.ReadCloser + c io.Closer +} + +func (rfcb *readFirstCloseBoth) Read(b []byte) (nr int, err error) { + return rfcb.r.Read(b) +} +func (rfcb *readFirstCloseBoth) Close() error { + err1 := rfcb.r.Close() + err2 := rfcb.c.Close() + if err1 != nil && err2 != nil { + return errors.New(err1.Error() + ", " + err2.Error()) + } + if err1 != nil { + return err1 + } + return err2 +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html_test.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html_test.go new file mode 100644 index 000000000000..9c876f75224d --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/html/html_test.go @@ -0,0 +1,60 @@ +package goproxy_html_test + +import ( + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/html" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +type ConstantServer int + +func (s ConstantServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=iso-8859-8") + //w.Header().Set("Content-Type","text/plain; charset=cp-1255") + w.Write([]byte{0xe3, 0xf3}) +} + +func TestCharset(t *testing.T) { + s := httptest.NewServer(ConstantServer(1)) + defer s.Close() + + ch := make(chan string, 2) + proxy := goproxy.NewProxyHttpServer() + proxy.OnResponse().Do(goproxy_html.HandleString( + func(s string, ctx *goproxy.ProxyCtx) string { + ch <- s + return s + })) + proxyServer := httptest.NewServer(proxy) + defer proxyServer.Close() + + proxyUrl, _ := url.Parse(proxyServer.URL) + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} + + resp, err := client.Get(s.URL + "/cp1255.txt") + if err != nil { + t.Fatal("GET:", err) + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("readAll:", err) + } + resp.Body.Close() + + inHandleString := "" + select { + case inHandleString = <-ch: + default: + } + + if len(b) != 2 || b[0] != 0xe3 || b[1] != 0xf3 { + t.Error("Did not translate back to 0xe3,0xf3, instead", b) + } + if inHandleString != "דף" { + t.Error("HandleString did not convert DALET & PEH SOFIT (דף) from ISO-8859-8 to utf-8, got", []byte(inHandleString)) + } +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/image/image.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/image/image.go new file mode 100644 index 000000000000..3dc26ff3747b --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/ext/image/image.go @@ -0,0 +1,78 @@ +package goproxy_image + +import ( + "bytes" + "image" + _ "image/gif" + "image/jpeg" + "image/png" + "io/ioutil" + "net/http" + . "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/regretable" +) + +var RespIsImage = ContentTypeIs("image/gif", + "image/jpeg", + "image/pjpeg", + "application/octet-stream", + "image/png") + +// "image/tiff" tiff support is in external package, and rarely used, so we omitted it + +func HandleImage(f func(img image.Image, ctx *ProxyCtx) image.Image) RespHandler { + return FuncRespHandler(func(resp *http.Response, ctx *ProxyCtx) *http.Response { + if !RespIsImage.HandleResp(resp, ctx) { + return resp + } + if resp.StatusCode != 200 { + // we might get 304 - not modified response without data + return resp + } + contentType := resp.Header.Get("Content-Type") + + const kb = 1024 + regret := regretable.NewRegretableReaderCloserSize(resp.Body, 16*kb) + resp.Body = regret + img, imgType, err := image.Decode(resp.Body) + if err != nil { + regret.Regret() + ctx.Warnf("%s: %s", ctx.Req.Method+" "+ctx.Req.URL.String()+" Image from "+ctx.Req.RequestURI+"content type"+ + contentType+"cannot be decoded returning original image", err) + return resp + } + result := f(img, ctx) + buf := bytes.NewBuffer([]byte{}) + switch contentType { + // No gif image encoder in go - convert to png + case "image/gif", "image/png": + if err := png.Encode(buf, result); err != nil { + ctx.Warnf("Cannot encode image, returning orig %v %v", ctx.Req.URL.String(), err) + return resp + } + resp.Header.Set("Content-Type", "image/png") + case "image/jpeg", "image/pjpeg": + if err := jpeg.Encode(buf, result, nil); err != nil { + ctx.Warnf("Cannot encode image, returning orig %v %v", ctx.Req.URL.String(), err) + return resp + } + case "application/octet-stream": + switch imgType { + case "jpeg": + if err := jpeg.Encode(buf, result, nil); err != nil { + ctx.Warnf("Cannot encode image as jpeg, returning orig %v %v", ctx.Req.URL.String(), err) + return resp + } + case "png", "gif": + if err := png.Encode(buf, result); err != nil { + ctx.Warnf("Cannot encode image as png, returning orig %v %v", ctx.Req.URL.String(), err) + return resp + } + } + default: + panic("unhandlable type" + contentType) + } + resp.Body = ioutil.NopCloser(buf) + return resp + }) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/https.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/https.go new file mode 100644 index 000000000000..2ff4d6a7b85d --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/https.go @@ -0,0 +1,366 @@ +package goproxy + +import ( + "bufio" + "crypto/tls" + "errors" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync/atomic" +) + +type ConnectActionLiteral int + +const ( + ConnectAccept = iota + ConnectReject + ConnectMitm + ConnectHijack + ConnectHTTPMitm +) + +var ( + OkConnect = &ConnectAction{Action: ConnectAccept, TLSConfig: TLSConfigFromCA(&GoproxyCa)} + MitmConnect = &ConnectAction{Action: ConnectMitm, TLSConfig: TLSConfigFromCA(&GoproxyCa)} + HTTPMitmConnect = &ConnectAction{Action: ConnectHTTPMitm, TLSConfig: TLSConfigFromCA(&GoproxyCa)} + RejectConnect = &ConnectAction{Action: ConnectReject, TLSConfig: TLSConfigFromCA(&GoproxyCa)} +) + +type ConnectAction struct { + Action ConnectActionLiteral + Hijack func(req *http.Request, client net.Conn, ctx *ProxyCtx) + TLSConfig func(host string, ctx *ProxyCtx) (*tls.Config, error) +} + +func stripPort(s string) string { + ix := strings.IndexRune(s, ':') + if ix == -1 { + return s + } + return s[:ix] +} + +func (proxy *ProxyHttpServer) dial(network, addr string) (c net.Conn, err error) { + if proxy.Tr.Dial != nil { + return proxy.Tr.Dial(network, addr) + } + return net.Dial(network, addr) +} + +func (proxy *ProxyHttpServer) connectDial(network, addr string) (c net.Conn, err error) { + if proxy.ConnectDial == nil { + return proxy.dial(network, addr) + } + return proxy.ConnectDial(network, addr) +} + +func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request) { + ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), proxy: proxy} + + hij, ok := w.(http.Hijacker) + if !ok { + panic("httpserver does not support hijacking") + } + + proxyClient, _, e := hij.Hijack() + if e != nil { + panic("Cannot hijack connection " + e.Error()) + } + + ctx.Logf("Running %d CONNECT handlers", len(proxy.httpsHandlers)) + todo, host := OkConnect, r.URL.Host + for i, h := range proxy.httpsHandlers { + newtodo, newhost := h.HandleConnect(host, ctx) + + // If found a result, break the loop immediately + if newtodo != nil { + todo, host = newtodo, newhost + ctx.Logf("on %dth handler: %v %s", i, todo, host) + break + } + } + switch todo.Action { + case ConnectAccept: + if !hasPort.MatchString(host) { + host += ":80" + } + targetSiteCon, err := proxy.connectDial("tcp", host) + if err != nil { + httpError(proxyClient, ctx, err) + return + } + ctx.Logf("Accepting CONNECT to %s", host) + proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) + go copyAndClose(ctx, targetSiteCon, proxyClient) + go copyAndClose(ctx, proxyClient, targetSiteCon) + case ConnectHijack: + ctx.Logf("Hijacking CONNECT to %s", host) + proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) + todo.Hijack(r, proxyClient, ctx) + case ConnectHTTPMitm: + proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) + ctx.Logf("Assuming CONNECT is plain HTTP tunneling, mitm proxying it") + targetSiteCon, err := proxy.connectDial("tcp", host) + if err != nil { + ctx.Warnf("Error dialing to %s: %s", host, err.Error()) + return + } + for { + client := bufio.NewReader(proxyClient) + remote := bufio.NewReader(targetSiteCon) + req, err := http.ReadRequest(client) + if err != nil && err != io.EOF { + ctx.Warnf("cannot read request of MITM HTTP client: %+#v", err) + } + if err != nil { + return + } + req, resp := proxy.filterRequest(req, ctx) + if resp == nil { + if err := req.Write(targetSiteCon); err != nil { + httpError(proxyClient, ctx, err) + return + } + resp, err = http.ReadResponse(remote, req) + if err != nil { + httpError(proxyClient, ctx, err) + return + } + } + resp = proxy.filterResponse(resp, ctx) + if err := resp.Write(proxyClient); err != nil { + httpError(proxyClient, ctx, err) + return + } + } + case ConnectMitm: + proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) + ctx.Logf("Assuming CONNECT is TLS, mitm proxying it") + // this goes in a separate goroutine, so that the net/http server won't think we're + // still handling the request even after hijacking the connection. Those HTTP CONNECT + // request can take forever, and the server will be stuck when "closed". + // TODO: Allow Server.Close() mechanism to shut down this connection as nicely as possible + tlsConfig := defaultTLSConfig + if todo.TLSConfig != nil { + var err error + tlsConfig, err = todo.TLSConfig(host, ctx) + if err != nil { + httpError(proxyClient, ctx, err) + return + } + } + go func() { + //TODO: cache connections to the remote website + rawClientTls := tls.Server(proxyClient, tlsConfig) + if err := rawClientTls.Handshake(); err != nil { + ctx.Warnf("Cannot handshake client %v %v", r.Host, err) + return + } + defer rawClientTls.Close() + clientTlsReader := bufio.NewReader(rawClientTls) + for !isEof(clientTlsReader) { + req, err := http.ReadRequest(clientTlsReader) + if err != nil && err != io.EOF { + return + } + if err != nil { + ctx.Warnf("Cannot read TLS request from mitm'd client %v %v", r.Host, err) + return + } + req.RemoteAddr = r.RemoteAddr // since we're converting the request, need to carry over the original connecting IP as well + ctx.Logf("req %v", r.Host) + req.URL, err = url.Parse("https://" + r.Host + req.URL.String()) + + // Bug fix which goproxy fails to provide request + // information URL in the context when does HTTPS MITM + ctx.Req = req + + req, resp := proxy.filterRequest(req, ctx) + if resp == nil { + if err != nil { + ctx.Warnf("Illegal URL %s", "https://"+r.Host+req.URL.Path) + return + } + removeProxyHeaders(ctx, req) + resp, err = ctx.RoundTrip(req) + if err != nil { + ctx.Warnf("Cannot read TLS response from mitm'd server %v", err) + return + } + ctx.Logf("resp %v", resp.Status) + } + resp = proxy.filterResponse(resp, ctx) + text := resp.Status + statusCode := strconv.Itoa(resp.StatusCode) + " " + if strings.HasPrefix(text, statusCode) { + text = text[len(statusCode):] + } + // always use 1.1 to support chunked encoding + if _, err := io.WriteString(rawClientTls, "HTTP/1.1"+" "+statusCode+text+"\r\n"); err != nil { + ctx.Warnf("Cannot write TLS response HTTP status from mitm'd client: %v", err) + return + } + // Since we don't know the length of resp, return chunked encoded response + // TODO: use a more reasonable scheme + resp.Header.Del("Content-Length") + resp.Header.Set("Transfer-Encoding", "chunked") + if err := resp.Header.Write(rawClientTls); err != nil { + ctx.Warnf("Cannot write TLS response header from mitm'd client: %v", err) + return + } + if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil { + ctx.Warnf("Cannot write TLS response header end from mitm'd client: %v", err) + return + } + chunked := newChunkedWriter(rawClientTls) + if _, err := io.Copy(chunked, resp.Body); err != nil { + ctx.Warnf("Cannot write TLS response body from mitm'd client: %v", err) + return + } + if err := chunked.Close(); err != nil { + ctx.Warnf("Cannot write TLS chunked EOF from mitm'd client: %v", err) + return + } + if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil { + ctx.Warnf("Cannot write TLS response chunked trailer from mitm'd client: %v", err) + return + } + } + ctx.Logf("Exiting on EOF") + }() + case ConnectReject: + if ctx.Resp != nil { + if err := ctx.Resp.Write(proxyClient); err != nil { + ctx.Warnf("Cannot write response that reject http CONNECT: %v", err) + } + } + proxyClient.Close() + } +} + +func httpError(w io.WriteCloser, ctx *ProxyCtx, err error) { + if _, err := io.WriteString(w, "HTTP/1.1 502 Bad Gateway\r\n\r\n"); err != nil { + ctx.Warnf("Error responding to client: %s", err) + } + if err := w.Close(); err != nil { + ctx.Warnf("Error closing client connection: %s", err) + } +} + +func copyAndClose(ctx *ProxyCtx, w, r net.Conn) { + connOk := true + if _, err := io.Copy(w, r); err != nil { + connOk = false + ctx.Warnf("Error copying to client: %s", err) + } + if err := r.Close(); err != nil && connOk { + ctx.Warnf("Error closing: %s", err) + } +} + +func dialerFromEnv(proxy *ProxyHttpServer) func(network, addr string) (net.Conn, error) { + https_proxy := os.Getenv("HTTPS_PROXY") + if https_proxy == "" { + https_proxy = os.Getenv("https_proxy") + } + if https_proxy == "" { + return nil + } + return proxy.NewConnectDialToProxy(https_proxy) +} + +func (proxy *ProxyHttpServer) NewConnectDialToProxy(https_proxy string) func(network, addr string) (net.Conn, error) { + u, err := url.Parse(https_proxy) + if err != nil { + return nil + } + if u.Scheme == "" || u.Scheme == "http" { + if strings.IndexRune(u.Host, ':') == -1 { + u.Host += ":80" + } + return func(network, addr string) (net.Conn, error) { + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: make(http.Header), + } + c, err := proxy.dial(network, u.Host) + if err != nil { + return nil, err + } + connectReq.Write(c) + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(c) + resp, err := http.ReadResponse(br, connectReq) + if err != nil { + c.Close() + return nil, err + } + if resp.StatusCode != 200 { + resp, _ := ioutil.ReadAll(resp.Body) + c.Close() + return nil, errors.New("proxy refused connection" + string(resp)) + } + return c, nil + } + } + if u.Scheme == "https" { + if strings.IndexRune(u.Host, ':') == -1 { + u.Host += ":443" + } + return func(network, addr string) (net.Conn, error) { + c, err := proxy.dial(network, u.Host) + if err != nil { + return nil, err + } + c = tls.Client(c, proxy.Tr.TLSClientConfig) + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: make(http.Header), + } + connectReq.Write(c) + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(c) + resp, err := http.ReadResponse(br, connectReq) + if err != nil { + c.Close() + return nil, err + } + if resp.StatusCode != 200 { + body, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 500)) + resp.Body.Close() + c.Close() + return nil, errors.New("proxy refused connection" + string(body)) + } + return c, nil + } + } + return nil +} + +func TLSConfigFromCA(ca *tls.Certificate) func(host string, ctx *ProxyCtx) (*tls.Config, error) { + return func(host string, ctx *ProxyCtx) (*tls.Config, error) { + config := *defaultTLSConfig + ctx.Logf("signing for %s", stripPort(host)) + cert, err := signHost(*ca, []string{stripPort(host)}) + if err != nil { + ctx.Warnf("Cannot sign host certificate with provided CA: %s", err) + return nil, err + } + config.Certificates = append(config.Certificates, cert) + return &config, nil + } +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/key.pem b/Godeps/_workspace/src/github.com/elazarl/goproxy/key.pem new file mode 100644 index 000000000000..2438b376032a --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/key.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQC/P0FsJomPGzvdO9yreV4/faEAZ6tDVGC+VnrxnidmahUd+X7Y +2v+bR2Zb4Z05+lNyz8rN8mNgav/zjHnbh+K5HwZ1nQc61cnPIXmx6hadsEi7KvU9 +sSmBGEZAyqo5S6NgTF4tt80c8ignxdnVXPK/djGNuaNYD5L+4570da0NswIDAQAB +AoGBALzIv1b4D7ARTR3NOr6V9wArjiOtMjUrdLhO+9vIp9IEA8ZsA9gjDlCEwbkP +VDnoLjnWfraff5Os6+3JjHy1fYpUiCdnk2XA6iJSL1XWKQZPt3wOunxP4lalDgED +QTRReFbA/y/Z4kSfTXpVj68ytcvSRW/N7q5/qRtbN9804jpBAkEA0s6lvH2btSLA +mcEdwhs7zAslLbdld7rvfUeP82gPPk0S6yUqTNyikqshM9AwAktHY7WvYdKl+ghZ +HTxKVC4DoQJBAOg/IAW5RbXknP+Lf7AVtBgw3E+Yfa3mcdLySe8hjxxyZq825Zmu +Rt5Qj4Lw6ifSFNy4kiiSpE/ZCukYvUXGENMCQFkPxSWlS6tzSzuqQxBGwTSrYMG3 +wb6b06JyIXcMd6Qym9OMmBpw/J5KfnSNeDr/4uFVWQtTG5xO+pdHaX+3EQECQQDl +qcbY4iX1gWVfr2tNjajSYz751yoxVbkpiT9joiQLVXYFvpu+JYEfRzsjmWl0h2Lq +AftG8/xYmaEYcMZ6wSrRAkBUwiom98/8wZVlB6qbwhU1EKDFANvICGSWMIhPx3v7 +MJqTIj4uJhte2/uyVvZ6DC6noWYgy+kLgqG0S97tUEG8 +-----END RSA PRIVATE KEY----- diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/proxy.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/proxy.go new file mode 100644 index 000000000000..e4ed0600ced9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/proxy.go @@ -0,0 +1,162 @@ +package goproxy + +import ( + "bufio" + "io" + "log" + "net" + "net/http" + "os" + "regexp" + "sync/atomic" +) + +// The basic proxy type. Implements http.Handler. +type ProxyHttpServer struct { + // session variable must be aligned in i386 + // see http://golang.org/src/pkg/sync/atomic/doc.go#L41 + sess int64 + // setting Verbose to true will log information on each request sent to the proxy + Verbose bool + Logger *log.Logger + NonproxyHandler http.Handler + reqHandlers []ReqHandler + respHandlers []RespHandler + httpsHandlers []HttpsHandler + Tr *http.Transport + // ConnectDial will be used to create TCP connections for CONNECT requests + // if nil Tr.Dial will be used + ConnectDial func(network string, addr string) (net.Conn, error) +} + +var hasPort = regexp.MustCompile(`:\d+$`) + +func copyHeaders(dst, src http.Header) { + for k, _ := range dst { + dst.Del(k) + } + for k, vs := range src { + for _, v := range vs { + dst.Add(k, v) + } + } +} + +func isEof(r *bufio.Reader) bool { + _, err := r.Peek(1) + if err == io.EOF { + return true + } + return false +} + +func (proxy *ProxyHttpServer) filterRequest(r *http.Request, ctx *ProxyCtx) (req *http.Request, resp *http.Response) { + req = r + for _, h := range proxy.reqHandlers { + req, resp = h.Handle(r, ctx) + // non-nil resp means the handler decided to skip sending the request + // and return canned response instead. + if resp != nil { + break + } + } + return +} +func (proxy *ProxyHttpServer) filterResponse(respOrig *http.Response, ctx *ProxyCtx) (resp *http.Response) { + resp = respOrig + for _, h := range proxy.respHandlers { + ctx.Resp = resp + resp = h.Handle(resp, ctx) + } + return +} + +func removeProxyHeaders(ctx *ProxyCtx, r *http.Request) { + r.RequestURI = "" // this must be reset when serving a request with the client + ctx.Logf("Sending request %v %v", r.Method, r.URL.String()) + // If no Accept-Encoding header exists, Transport will add the headers it can accept + // and would wrap the response body with the relevant reader. + r.Header.Del("Accept-Encoding") + // curl can add that, see + // http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/web-proxy-connection-header.html + r.Header.Del("Proxy-Connection") + r.Header.Del("Proxy-Authenticate") + r.Header.Del("Proxy-Authorization") + // Connection, Authenticate and Authorization are single hop Header: + // http://www.w3.org/Protocols/rfc2616/rfc2616.txt + // 14.10 Connection + // The Connection general-header field allows the sender to specify + // options that are desired for that particular connection and MUST NOT + // be communicated by proxies over further connections. + r.Header.Del("Connection") +} + +// Standard net/http function. Shouldn't be used directly, http.Serve will use it. +func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + //r.Header["X-Forwarded-For"] = w.RemoteAddr() + if r.Method == "CONNECT" { + proxy.handleHttps(w, r) + } else { + ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), proxy: proxy} + + var err error + ctx.Logf("Got request %v %v %v %v", r.URL.Path, r.Host, r.Method, r.URL.String()) + if !r.URL.IsAbs() { + proxy.NonproxyHandler.ServeHTTP(w, r) + return + } + r, resp := proxy.filterRequest(r, ctx) + + if resp == nil { + removeProxyHeaders(ctx, r) + resp, err = ctx.RoundTrip(r) + if err != nil { + ctx.Error = err + resp = proxy.filterResponse(nil, ctx) + if resp == nil { + ctx.Logf("error read response %v %v:", r.URL.Host, err.Error()) + http.Error(w, err.Error(), 500) + return + } + } + ctx.Logf("Received response %v", resp.Status) + } + origBody := resp.Body + resp = proxy.filterResponse(resp, ctx) + + ctx.Logf("Copying response to client %v [%d]", resp.Status, resp.StatusCode) + // http.ResponseWriter will take care of filling the correct response length + // Setting it now, might impose wrong value, contradicting the actual new + // body the user returned. + // We keep the original body to remove the header only if things changed. + // This will prevent problems with HEAD requests where there's no body, yet, + // the Content-Length header should be set. + if origBody != resp.Body { + resp.Header.Del("Content-Length") + } + copyHeaders(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + nr, err := io.Copy(w, resp.Body) + if err := resp.Body.Close(); err != nil { + ctx.Warnf("Can't close response body %v", err) + } + ctx.Logf("Copied %v bytes to client error=%v", nr, err) + } +} + +// New proxy server, logs to StdErr by default +func NewProxyHttpServer() *ProxyHttpServer { + proxy := ProxyHttpServer{ + Logger: log.New(os.Stderr, "", log.LstdFlags), + reqHandlers: []ReqHandler{}, + respHandlers: []RespHandler{}, + httpsHandlers: []HttpsHandler{}, + NonproxyHandler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Error(w, "This is a proxy server. Does not respond to non-proxy requests.", 500) + }), + Tr: &http.Transport{TLSClientConfig: tlsClientSkipVerify, + Proxy: http.ProxyFromEnvironment}, + } + proxy.ConnectDial = dialerFromEnv(&proxy) + return &proxy +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/proxy_test.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/proxy_test.go new file mode 100644 index 000000000000..8b147cfd04ef --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/proxy_test.go @@ -0,0 +1,767 @@ +package goproxy_test + +import ( + "bufio" + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "image" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "strings" + "testing" + + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/image" +) + +var acceptAllCerts = &tls.Config{InsecureSkipVerify: true} + +var noProxyClient = &http.Client{Transport: &http.Transport{TLSClientConfig: acceptAllCerts}} + +var https = httptest.NewTLSServer(nil) +var srv = httptest.NewServer(nil) +var fs = httptest.NewServer(http.FileServer(http.Dir("."))) + +type QueryHandler struct{} + +func (QueryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if err := req.ParseForm(); err != nil { + panic(err) + } + io.WriteString(w, req.Form.Get("result")) +} + +func init() { + http.DefaultServeMux.Handle("/bobo", ConstantHanlder("bobo")) + http.DefaultServeMux.Handle("/query", QueryHandler{}) +} + +type ConstantHanlder string + +func (h ConstantHanlder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, string(h)) +} + +func get(url string, client *http.Client) ([]byte, error) { + resp, err := client.Get(url) + if err != nil { + return nil, err + } + txt, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return nil, err + } + return txt, nil +} + +func getOrFail(url string, client *http.Client, t *testing.T) []byte { + txt, err := get(url, client) + if err != nil { + t.Fatal("Can't fetch url", url, err) + } + return txt +} + +func localFile(url string) string { return fs.URL + "/" + url } +func localTls(url string) string { return https.URL + url } + +func TestSimpleHttpReqWithProxy(t *testing.T) { + client, s := oneShotProxy(goproxy.NewProxyHttpServer(), t) + defer s.Close() + + if r := string(getOrFail(srv.URL+"/bobo", client, t)); r != "bobo" { + t.Error("proxy server does not serve constant handlers", r) + } + if r := string(getOrFail(srv.URL+"/bobo", client, t)); r != "bobo" { + t.Error("proxy server does not serve constant handlers", r) + } + + if string(getOrFail(https.URL+"/bobo", client, t)) != "bobo" { + t.Error("TLS server does not serve constant handlers, when proxy is used") + } +} + +func oneShotProxy(proxy *goproxy.ProxyHttpServer, t *testing.T) (client *http.Client, s *httptest.Server) { + s = httptest.NewServer(proxy) + + proxyUrl, _ := url.Parse(s.URL) + tr := &http.Transport{TLSClientConfig: acceptAllCerts, Proxy: http.ProxyURL(proxyUrl)} + client = &http.Client{Transport: tr} + return +} + +func TestSimpleHook(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest(goproxy.SrcIpIs("127.0.0.1")).DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + req.URL.Path = "/bobo" + return req, nil + }) + client, l := oneShotProxy(proxy, t) + defer l.Close() + + if result := string(getOrFail(srv.URL+("/momo"), client, t)); result != "bobo" { + t.Error("Redirecting all requests from 127.0.0.1 to bobo, didn't work." + + " (Might break if Go's client sets RemoteAddr to IPv6 address). Got: " + + result) + } +} + +func TestAlwaysHook(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + req.URL.Path = "/bobo" + return req, nil + }) + client, l := oneShotProxy(proxy, t) + defer l.Close() + + if result := string(getOrFail(srv.URL+("/momo"), client, t)); result != "bobo" { + t.Error("Redirecting all requests from 127.0.0.1 to bobo, didn't work." + + " (Might break if Go's client sets RemoteAddr to IPv6 address). Got: " + + result) + } +} + +func TestReplaceResponse(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + resp.StatusCode = http.StatusOK + resp.Body = ioutil.NopCloser(bytes.NewBufferString("chico")) + return resp + }) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + if result := string(getOrFail(srv.URL+("/momo"), client, t)); result != "chico" { + t.Error("hooked response, should be chico, instead:", result) + } +} + +func TestReplaceReponseForUrl(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnResponse(goproxy.UrlIs("/koko")).DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + resp.StatusCode = http.StatusOK + resp.Body = ioutil.NopCloser(bytes.NewBufferString("chico")) + return resp + }) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + if result := string(getOrFail(srv.URL+("/koko"), client, t)); result != "chico" { + t.Error("hooked 'koko', should be chico, instead:", result) + } + if result := string(getOrFail(srv.URL+("/bobo"), client, t)); result != "bobo" { + t.Error("still, bobo should stay as usual, instead:", result) + } +} + +func TestOneShotFileServer(t *testing.T) { + client, l := oneShotProxy(goproxy.NewProxyHttpServer(), t) + defer l.Close() + + file := "test_data/panda.png" + info, err := os.Stat(file) + if err != nil { + t.Fatal("Cannot find", file) + } + if resp, err := client.Get(fs.URL + "/" + file); err == nil { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("got", string(b)) + } + if int64(len(b)) != info.Size() { + t.Error("Expected Length", file, info.Size(), "actually", len(b), "starts", string(b[:10])) + } + } else { + t.Fatal("Cannot read from fs server", err) + } +} + +func TestContentType(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnResponse(goproxy.ContentTypeIs("image/png")).DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + resp.Header.Set("X-Shmoopi", "1") + return resp + }) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + for _, file := range []string{"test_data/panda.png", "test_data/football.png"} { + if resp, err := client.Get(localFile(file)); err != nil || resp.Header.Get("X-Shmoopi") != "1" { + if err == nil { + t.Error("pngs should have X-Shmoopi header = 1, actually", resp.Header.Get("X-Shmoopi")) + } else { + t.Error("error reading png", err) + } + } + } + + file := "baby.jpg" + if resp, err := client.Get(localFile(file)); err != nil || resp.Header.Get("X-Shmoopi") != "" { + if err == nil { + t.Error("Non png images should NOT have X-Shmoopi header at all", resp.Header.Get("X-Shmoopi")) + } else { + t.Error("error reading png", err) + } + } +} + +func getImage(file string, t *testing.T) image.Image { + newimage, err := ioutil.ReadFile(file) + if err != nil { + t.Fatal("Cannot read file", file, err) + } + img, _, err := image.Decode(bytes.NewReader(newimage)) + if err != nil { + t.Fatal("Cannot decode image", file, err) + } + return img +} + +func readAll(r io.Reader, t *testing.T) []byte { + b, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal("Cannot read", err) + } + return b +} +func readFile(file string, t *testing.T) []byte { + b, err := ioutil.ReadFile(file) + if err != nil { + t.Fatal("Cannot read", err) + } + return b +} +func fatalOnErr(err error, msg string, t *testing.T) { + if err != nil { + t.Fatal(msg, err) + } +} +func panicOnErr(err error, msg string) { + if err != nil { + println(err.Error() + ":-" + msg) + os.Exit(-1) + } +} + +func compareImage(eImg, aImg image.Image, t *testing.T) { + if eImg.Bounds().Dx() != aImg.Bounds().Dx() || eImg.Bounds().Dy() != aImg.Bounds().Dy() { + t.Error("image sizes different") + return + } + for i := 0; i < eImg.Bounds().Dx(); i++ { + for j := 0; j < eImg.Bounds().Dy(); j++ { + er, eg, eb, ea := eImg.At(i, j).RGBA() + ar, ag, ab, aa := aImg.At(i, j).RGBA() + if er != ar || eg != ag || eb != ab || ea != aa { + t.Error("images different at", i, j, "vals\n", er, eg, eb, ea, "\n", ar, ag, ab, aa, aa) + return + } + } + } +} + +func TestConstantImageHandler(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + //panda := getImage("panda.png", t) + football := getImage("test_data/football.png", t) + proxy.OnResponse().Do(goproxy_image.HandleImage(func(img image.Image, ctx *goproxy.ProxyCtx) image.Image { + return football + })) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + resp, err := client.Get(localFile("test_data/panda.png")) + if err != nil { + t.Fatal("Cannot get panda.png", err) + } + + img, _, err := image.Decode(resp.Body) + if err != nil { + t.Error("decode", err) + } else { + compareImage(football, img, t) + } +} + +func TestImageHandler(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + football := getImage("test_data/football.png", t) + + proxy.OnResponse(goproxy.UrlIs("/test_data/panda.png")).Do(goproxy_image.HandleImage(func(img image.Image, ctx *goproxy.ProxyCtx) image.Image { + return football + })) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + resp, err := client.Get(localFile("test_data/panda.png")) + if err != nil { + t.Fatal("Cannot get panda.png", err) + } + + img, _, err := image.Decode(resp.Body) + if err != nil { + t.Error("decode", err) + } else { + compareImage(football, img, t) + } + + // and again + resp, err = client.Get(localFile("test_data/panda.png")) + if err != nil { + t.Fatal("Cannot get panda.png", err) + } + + img, _, err = image.Decode(resp.Body) + if err != nil { + t.Error("decode", err) + } else { + compareImage(football, img, t) + } +} + +func TestChangeResp(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + resp.Body.Read([]byte{0}) + resp.Body = ioutil.NopCloser(new(bytes.Buffer)) + return resp + }) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + resp, err := client.Get(localFile("test_data/panda.png")) + if err != nil { + t.Fatal(err) + } + ioutil.ReadAll(resp.Body) + _, err = client.Get(localFile("/bobo")) + if err != nil { + t.Fatal(err) + } +} +func TestReplaceImage(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + + panda := getImage("test_data/panda.png", t) + football := getImage("test_data/football.png", t) + + proxy.OnResponse(goproxy.UrlIs("/test_data/panda.png")).Do(goproxy_image.HandleImage(func(img image.Image, ctx *goproxy.ProxyCtx) image.Image { + return football + })) + proxy.OnResponse(goproxy.UrlIs("/test_data/football.png")).Do(goproxy_image.HandleImage(func(img image.Image, ctx *goproxy.ProxyCtx) image.Image { + return panda + })) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + imgByPandaReq, _, err := image.Decode(bytes.NewReader(getOrFail(localFile("test_data/panda.png"), client, t))) + fatalOnErr(err, "decode panda", t) + compareImage(football, imgByPandaReq, t) + + imgByFootballReq, _, err := image.Decode(bytes.NewReader(getOrFail(localFile("test_data/football.png"), client, t))) + fatalOnErr(err, "decode football", t) + compareImage(panda, imgByFootballReq, t) +} + +func getCert(c *tls.Conn, t *testing.T) []byte { + if err := c.Handshake(); err != nil { + t.Fatal("cannot handshake", err) + } + return c.ConnectionState().PeerCertificates[0].Raw +} + +func TestSimpleMitm(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest(goproxy.ReqHostIs(https.Listener.Addr().String())).HandleConnect(goproxy.AlwaysMitm) + proxy.OnRequest(goproxy.ReqHostIs("no such host exists")).HandleConnect(goproxy.AlwaysMitm) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + c, err := tls.Dial("tcp", https.Listener.Addr().String(), &tls.Config{InsecureSkipVerify: true}) + if err != nil { + t.Fatal("cannot dial to tcp server", err) + } + origCert := getCert(c, t) + c.Close() + + c2, err := net.Dial("tcp", l.Listener.Addr().String()) + if err != nil { + t.Fatal("dialing to proxy", err) + } + creq, err := http.NewRequest("CONNECT", https.URL, nil) + //creq,err := http.NewRequest("CONNECT","https://google.com:443",nil) + if err != nil { + t.Fatal("create new request", creq) + } + creq.Write(c2) + c2buf := bufio.NewReader(c2) + resp, err := http.ReadResponse(c2buf, creq) + if err != nil || resp.StatusCode != 200 { + t.Fatal("Cannot CONNECT through proxy", err) + } + c2tls := tls.Client(c2, &tls.Config{InsecureSkipVerify: true}) + proxyCert := getCert(c2tls, t) + + if bytes.Equal(proxyCert, origCert) { + t.Errorf("Certificate after mitm is not different\n%v\n%v", + base64.StdEncoding.EncodeToString(origCert), + base64.StdEncoding.EncodeToString(proxyCert)) + } + + if resp := string(getOrFail(https.URL+"/bobo", client, t)); resp != "bobo" { + t.Error("Wrong response when mitm", resp, "expected bobo") + } + if resp := string(getOrFail(https.URL+"/query?result=bar", client, t)); resp != "bar" { + t.Error("Wrong response when mitm", resp, "expected bar") + } +} + +func TestConnectHandler(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + althttps := httptest.NewTLSServer(ConstantHanlder("althttps")) + proxy.OnRequest().HandleConnectFunc(func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + u, _ := url.Parse(althttps.URL) + return goproxy.OkConnect, u.Host + }) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + if resp := string(getOrFail(https.URL+"/alturl", client, t)); resp != "althttps" { + t.Error("Proxy should redirect CONNECT requests to local althttps server, expected 'althttps' got ", resp) + } +} + +func TestMitmIsFiltered(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + //proxy.Verbose = true + proxy.OnRequest(goproxy.ReqHostIs(https.Listener.Addr().String())).HandleConnect(goproxy.AlwaysMitm) + proxy.OnRequest(goproxy.UrlIs("/momo")).DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + return nil, goproxy.TextResponse(req, "koko") + }) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + if resp := string(getOrFail(https.URL+"/momo", client, t)); resp != "koko" { + t.Error("Proxy should capture /momo to be koko and not", resp) + } + + if resp := string(getOrFail(https.URL+"/bobo", client, t)); resp != "bobo" { + t.Error("But still /bobo should be bobo and not", resp) + } +} + +func TestFirstHandlerMatches(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + return nil, goproxy.TextResponse(req, "koko") + }) + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + panic("should never get here, previous response is no null") + }) + + client, l := oneShotProxy(proxy, t) + defer l.Close() + + if resp := string(getOrFail(srv.URL+"/", client, t)); resp != "koko" { + t.Error("should return always koko and not", resp) + } +} + +func constantHttpServer(content []byte) (addr string) { + l, err := net.Listen("tcp", "localhost:0") + panicOnErr(err, "listen") + go func() { + c, err := l.Accept() + panicOnErr(err, "accept") + buf := bufio.NewReader(c) + _, err = http.ReadRequest(buf) + panicOnErr(err, "readReq") + c.Write(content) + c.Close() + l.Close() + }() + return l.Addr().String() +} + +func TestIcyResponse(t *testing.T) { + // TODO: fix this test + return // skip for now + s := constantHttpServer([]byte("ICY 200 OK\r\n\r\nblablabla")) + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + _, l := oneShotProxy(proxy, t) + defer l.Close() + req, err := http.NewRequest("GET", "http://"+s, nil) + panicOnErr(err, "newReq") + proxyip := l.URL[len("http://"):] + println("got ip: " + proxyip) + c, err := net.Dial("tcp", proxyip) + panicOnErr(err, "dial") + defer c.Close() + req.WriteProxy(c) + raw, err := ioutil.ReadAll(c) + panicOnErr(err, "readAll") + if string(raw) != "ICY 200 OK\r\n\r\nblablabla" { + t.Error("Proxy did not send the malformed response received") + } +} + +type VerifyNoProxyHeaders struct { + *testing.T +} + +func (v VerifyNoProxyHeaders) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Connection") != "" || r.Header.Get("Proxy-Connection") != "" || + r.Header.Get("Proxy-Authenticate") != "" || r.Header.Get("Proxy-Authorization") != "" { + v.Error("Got Connection header from goproxy", r.Header) + } +} + +func TestNoProxyHeaders(t *testing.T) { + s := httptest.NewServer(VerifyNoProxyHeaders{t}) + client, l := oneShotProxy(goproxy.NewProxyHttpServer(), t) + defer l.Close() + req, err := http.NewRequest("GET", s.URL, nil) + panicOnErr(err, "bad request") + req.Header.Add("Connection", "close") + req.Header.Add("Proxy-Connection", "close") + req.Header.Add("Proxy-Authenticate", "auth") + req.Header.Add("Proxy-Authorization", "auth") + client.Do(req) +} + +func TestNoProxyHeadersHttps(t *testing.T) { + s := httptest.NewTLSServer(VerifyNoProxyHeaders{t}) + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) + client, l := oneShotProxy(proxy, t) + defer l.Close() + req, err := http.NewRequest("GET", s.URL, nil) + panicOnErr(err, "bad request") + req.Header.Add("Connection", "close") + req.Header.Add("Proxy-Connection", "close") + client.Do(req) +} + +func TestHeadReqHasContentLength(t *testing.T) { + client, l := oneShotProxy(goproxy.NewProxyHttpServer(), t) + defer l.Close() + + resp, err := client.Head(localFile("test_data/panda.png")) + panicOnErr(err, "resp to HEAD") + if resp.Header.Get("Content-Length") == "" { + t.Error("Content-Length should exist on HEAD requests") + } +} + +func TestChunkedResponse(t *testing.T) { + l, err := net.Listen("tcp", ":10234") + panicOnErr(err, "listen") + defer l.Close() + go func() { + for i := 0; i < 2; i++ { + c, err := l.Accept() + panicOnErr(err, "accept") + _, err = http.ReadRequest(bufio.NewReader(c)) + panicOnErr(err, "readrequest") + io.WriteString(c, "HTTP/1.1 200 OK\r\n"+ + "Content-Type: text/plain\r\n"+ + "Transfer-Encoding: chunked\r\n\r\n"+ + "25\r\n"+ + "This is the data in the first chunk\r\n\r\n"+ + "1C\r\n"+ + "and this is the second one\r\n\r\n"+ + "3\r\n"+ + "con\r\n"+ + "8\r\n"+ + "sequence\r\n0\r\n\r\n") + c.Close() + } + }() + + c, err := net.Dial("tcp", "localhost:10234") + panicOnErr(err, "dial") + defer c.Close() + req, _ := http.NewRequest("GET", "/", nil) + req.Write(c) + resp, err := http.ReadResponse(bufio.NewReader(c), req) + panicOnErr(err, "readresp") + b, err := ioutil.ReadAll(resp.Body) + panicOnErr(err, "readall") + expected := "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" + if string(b) != expected { + t.Errorf("Got `%v` expected `%v`", string(b), expected) + } + + proxy := goproxy.NewProxyHttpServer() + proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + panicOnErr(ctx.Error, "error reading output") + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + panicOnErr(err, "readall onresp") + if enc := resp.Header.Get("Transfer-Encoding"); enc != "" { + t.Fatal("Chunked response should be received as plaintext", enc) + } + resp.Body = ioutil.NopCloser(bytes.NewBufferString(strings.Replace(string(b), "e", "E", -1))) + return resp + }) + + client, s := oneShotProxy(proxy, t) + defer s.Close() + + resp, err = client.Get("http://localhost:10234/") + panicOnErr(err, "client.Get") + b, err = ioutil.ReadAll(resp.Body) + panicOnErr(err, "readall proxy") + if string(b) != strings.Replace(expected, "e", "E", -1) { + t.Error("expected", expected, "w/ e->E. Got", string(b)) + } +} + +func TestGoproxyThroughProxy(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy2 := goproxy.NewProxyHttpServer() + doubleString := func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + b, err := ioutil.ReadAll(resp.Body) + panicOnErr(err, "readAll resp") + resp.Body = ioutil.NopCloser(bytes.NewBufferString(string(b) + " " + string(b))) + return resp + } + proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) + proxy.OnResponse().DoFunc(doubleString) + + _, l := oneShotProxy(proxy, t) + defer l.Close() + + proxy2.ConnectDial = proxy2.NewConnectDialToProxy(l.URL) + + client, l2 := oneShotProxy(proxy2, t) + defer l2.Close() + if r := string(getOrFail(https.URL+"/bobo", client, t)); r != "bobo bobo" { + t.Error("Expected bobo doubled twice, got", r) + } + +} + +func TestGoproxyHijackConnect(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest(goproxy.ReqHostIs(srv.Listener.Addr().String())). + HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) { + t.Logf("URL %+#v\nSTR %s", req.URL, req.URL.String()) + resp, err := http.Get("http:" + req.URL.String() + "/bobo") + panicOnErr(err, "http.Get(CONNECT url)") + panicOnErr(resp.Write(client), "resp.Write(client)") + resp.Body.Close() + client.Close() + }) + client, l := oneShotProxy(proxy, t) + defer l.Close() + proxyAddr := l.Listener.Addr().String() + conn, err := net.Dial("tcp", proxyAddr) + panicOnErr(err, "conn "+proxyAddr) + buf := bufio.NewReader(conn) + writeConnect(conn) + readConnectResponse(buf) + if txt := readResponse(buf); txt != "bobo" { + t.Error("Expected bobo for CONNECT /foo, got", txt) + } + + if r := string(getOrFail(https.URL+"/bobo", client, t)); r != "bobo" { + t.Error("Expected bobo would keep working with CONNECT", r) + } +} + +func readResponse(buf *bufio.Reader) string { + req, err := http.NewRequest("GET", srv.URL, nil) + panicOnErr(err, "NewRequest") + resp, err := http.ReadResponse(buf, req) + panicOnErr(err, "resp.Read") + defer resp.Body.Close() + txt, err := ioutil.ReadAll(resp.Body) + panicOnErr(err, "resp.Read") + return string(txt) +} + +func writeConnect(w io.Writer) { + req, err := http.NewRequest("CONNECT", srv.URL[len("http://"):], nil) + panicOnErr(err, "NewRequest") + req.Write(w) + panicOnErr(err, "req(CONNECT).Write") +} + +func readConnectResponse(buf *bufio.Reader) { + _, err := buf.ReadString('\n') + panicOnErr(err, "resp.Read connect resp") + _, err = buf.ReadString('\n') + panicOnErr(err, "resp.Read connect resp") +} + +func TestCurlMinusP(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().HandleConnectFunc(func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + return goproxy.HTTPMitmConnect, host + }) + called := false + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + called = true + return req, nil + }) + _, l := oneShotProxy(proxy, t) + defer l.Close() + cmd := exec.Command("curl", "-p", "-sS", "--proxy", l.URL, srv.URL+"/bobo") + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(err) + } + if string(output) != "bobo" { + t.Error("Expected bobo, got", string(output)) + } + if !called { + t.Error("handler not called") + } +} + +func TestSelfRequest(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + _, l := oneShotProxy(proxy, t) + defer l.Close() + if !strings.Contains(string(getOrFail(l.URL, http.DefaultClient, t)), "non-proxy") { + t.Fatal("non proxy requests should fail") + } +} + +func TestHasGoproxyCA(t *testing.T) { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) + s := httptest.NewServer(proxy) + + proxyUrl, _ := url.Parse(s.URL) + goproxyCA := x509.NewCertPool() + goproxyCA.AddCert(goproxy.GoproxyCa.Leaf) + + tr := &http.Transport{TLSClientConfig: &tls.Config{RootCAs: goproxyCA}, Proxy: http.ProxyURL(proxyUrl)} + client := &http.Client{Transport: tr} + + if resp := string(getOrFail(https.URL+"/bobo", client, t)); resp != "bobo" { + t.Error("Wrong response when mitm", resp, "expected bobo") + } +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader.go new file mode 100644 index 000000000000..1458af587145 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader.go @@ -0,0 +1,97 @@ +package regretable + +import ( + "io" +) + +// A RegretableReader will allow you to read from a reader, and then +// to "regret" reading it, and push back everything you've read. +// For example, +// rb := NewRegretableReader(bytes.NewBuffer([]byte{1,2,3})) +// var b = make([]byte,1) +// rb.Read(b) // b[0] = 1 +// rb.Regret() +// ioutil.ReadAll(rb.Read) // returns []byte{1,2,3},nil +type RegretableReader struct { + reader io.Reader + overflow bool + r, w int + buf []byte +} + +var defaultBufferSize = 500 + +// Same as RegretableReader, but allows closing the underlying reader +type RegretableReaderCloser struct { + RegretableReader + c io.Closer +} + +// Closes the underlying readCloser, you cannot regret after closing the stream +func (rbc *RegretableReaderCloser) Close() error { + return rbc.c.Close() +} + +// initialize a RegretableReaderCloser with underlying readCloser rc +func NewRegretableReaderCloser(rc io.ReadCloser) *RegretableReaderCloser { + return &RegretableReaderCloser{*NewRegretableReader(rc), rc} +} + +// initialize a RegretableReaderCloser with underlying readCloser rc +func NewRegretableReaderCloserSize(rc io.ReadCloser, size int) *RegretableReaderCloser { + return &RegretableReaderCloser{*NewRegretableReaderSize(rc, size), rc} +} + +// The next read from the RegretableReader will be as if the underlying reader +// was never read (or from the last point forget is called). +func (rb *RegretableReader) Regret() { + if rb.overflow { + panic("regretting after overflow makes no sense") + } + rb.r = 0 +} + +// Will "forget" everything read so far. +// rb := NewRegretableReader(bytes.NewBuffer([]byte{1,2,3})) +// var b = make([]byte,1) +// rb.Read(b) // b[0] = 1 +// rb.Forget() +// rb.Read(b) // b[0] = 2 +// rb.Regret() +// ioutil.ReadAll(rb.Read) // returns []byte{2,3},nil +func (rb *RegretableReader) Forget() { + if rb.overflow { + panic("forgetting after overflow makes no sense") + } + rb.r = 0 + rb.w = 0 +} + +// initialize a RegretableReader with underlying reader r, whose buffer is size bytes long +func NewRegretableReaderSize(r io.Reader, size int) *RegretableReader { + return &RegretableReader{reader: r, buf: make([]byte, size) } +} + +// initialize a RegretableReader with underlying reader r +func NewRegretableReader(r io.Reader) *RegretableReader { + return NewRegretableReaderSize(r, defaultBufferSize) +} + +// reads from the underlying reader. Will buffer all input until Regret is called. +func (rb *RegretableReader) Read(p []byte) (n int, err error) { + if rb.overflow { + return rb.reader.Read(p) + } + if rb.r < rb.w { + n = copy(p, rb.buf[rb.r:rb.w]) + rb.r += n + return + } + n, err = rb.reader.Read(p) + bn := copy(rb.buf[rb.w:], p[:n]) + rb.w, rb.r = rb.w + bn, rb.w + n + if bn < n { + rb.overflow = true + } + return +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader_test.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader_test.go new file mode 100644 index 000000000000..55fa752bc808 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/regretable/regretreader_test.go @@ -0,0 +1,174 @@ +package regretable_test + +import ( + . "github.com/elazarl/goproxy/regretable" + "bytes" + "io" + "io/ioutil" + "strings" + "testing" +) + +func TestRegretableReader(t *testing.T) { + buf := new(bytes.Buffer) + mb := NewRegretableReader(buf) + word := "12345678" + buf.WriteString(word) + + fivebytes := make([]byte, 5) + mb.Read(fivebytes) + mb.Regret() + + s, _ := ioutil.ReadAll(mb) + if string(s) != word { + t.Errorf("Uncommited read is gone, [%d,%d] actual '%v' expected '%v'\n", len(s), len(word), string(s), word) + } +} + +func TestRegretableEmptyRead(t *testing.T) { + buf := new(bytes.Buffer) + mb := NewRegretableReader(buf) + word := "12345678" + buf.WriteString(word) + + zero := make([]byte, 0) + mb.Read(zero) + mb.Regret() + + s, err := ioutil.ReadAll(mb) + if string(s) != word { + t.Error("Uncommited read is gone, actual:", string(s), "expected:", word, "err:", err) + } +} + +func TestRegretableAlsoEmptyRead(t *testing.T) { + buf := new(bytes.Buffer) + mb := NewRegretableReader(buf) + word := "12345678" + buf.WriteString(word) + + one := make([]byte, 1) + zero := make([]byte, 0) + five := make([]byte, 5) + mb.Read(one) + mb.Read(zero) + mb.Read(five) + mb.Regret() + + s, _ := ioutil.ReadAll(mb) + if string(s) != word { + t.Error("Uncommited read is gone", string(s), "expected", word) + } +} + +func TestRegretableRegretBeforeRead(t *testing.T) { + buf := new(bytes.Buffer) + mb := NewRegretableReader(buf) + word := "12345678" + buf.WriteString(word) + + five := make([]byte, 5) + mb.Regret() + mb.Read(five) + + s, err := ioutil.ReadAll(mb) + if string(s) != "678" { + t.Error("Uncommited read is gone", string(s), len(string(s)), "expected", "678", len("678"), "err:", err) + } +} + +func TestRegretableFullRead(t *testing.T) { + buf := new(bytes.Buffer) + mb := NewRegretableReader(buf) + word := "12345678" + buf.WriteString(word) + + twenty := make([]byte, 20) + mb.Read(twenty) + mb.Regret() + + s, _ := ioutil.ReadAll(mb) + if string(s) != word { + t.Error("Uncommited read is gone", string(s), len(string(s)), "expected", word, len(word)) + } +} + +func assertEqual(t *testing.T, expected, actual string) { + if expected!=actual { + t.Fatal("Expected", expected, "actual", actual) + } +} + +func assertReadAll(t *testing.T, r io.Reader) string { + s, err := ioutil.ReadAll(r) + if err!=nil { + t.Fatal("error when reading", err) + } + return string(s) +} + +func TestRegretableRegretTwice(t *testing.T) { + buf := new(bytes.Buffer) + mb := NewRegretableReader(buf) + word := "12345678" + buf.WriteString(word) + + assertEqual(t, word, assertReadAll(t, mb)) + mb.Regret() + assertEqual(t, word, assertReadAll(t, mb)) + mb.Regret() + assertEqual(t, word, assertReadAll(t, mb)) +} + +type CloseCounter struct { + r io.Reader + closed int +} + +func (cc *CloseCounter) Read(b []byte) (int, error) { + return cc.r.Read(b) +} + +func (cc *CloseCounter) Close() error { + cc.closed++ + return nil +} + +func assert(t *testing.T, b bool, msg string) { + if !b { + t.Errorf("Assertion Error: %s", msg) + } +} + +func TestRegretableCloserSizeRegrets(t *testing.T) { + defer func() { + if r := recover(); r == nil || !strings.Contains(r.(string), "regret") { + t.Error("Did not panic when regretting overread buffer:", r) + } + }() + buf := new(bytes.Buffer) + buf.WriteString("123456") + mb := NewRegretableReaderCloserSize(ioutil.NopCloser(buf), 3) + mb.Read(make([]byte, 4)) + mb.Regret() +} + +func TestRegretableCloserRegretsClose(t *testing.T) { + buf := new(bytes.Buffer) + cc := &CloseCounter{buf, 0} + mb := NewRegretableReaderCloser(cc) + word := "12345678" + buf.WriteString(word) + + mb.Read([]byte{0}) + mb.Close() + if cc.closed != 1 { + t.Error("RegretableReaderCloser ignores Close") + } + mb.Regret() + mb.Close() + if cc.closed != 2 { + t.Error("RegretableReaderCloser does ignore Close after regret") + } + // TODO(elazar): return an error if client issues Close more than once after regret +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/responses.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/responses.go new file mode 100644 index 000000000000..b304b8829be9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/responses.go @@ -0,0 +1,38 @@ +package goproxy + +import ( + "bytes" + "io/ioutil" + "net/http" +) + +// Will generate a valid http response to the given request the response will have +// the given contentType, and http status. +// Typical usage, refuse to process requests to local addresses: +// +// proxy.OnRequest(IsLocalHost()).DoFunc(func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request,*http.Response) { +// return nil,NewResponse(r,goproxy.ContentTypeHtml,http.StatusUnauthorized, +// `Can't use proxy for local addresses`) +// }) +func NewResponse(r *http.Request, contentType string, status int, body string) *http.Response { + resp := &http.Response{} + resp.Request = r + resp.TransferEncoding = r.TransferEncoding + resp.Header = make(http.Header) + resp.Header.Add("Content-Type", contentType) + resp.StatusCode = status + buf := bytes.NewBufferString(body) + resp.ContentLength = int64(buf.Len()) + resp.Body = ioutil.NopCloser(buf) + return resp +} + +const ( + ContentTypeText = "text/plain" + ContentTypeHtml = "text/html" +) + +// Alias for NewResponse(r,ContentTypeText,http.StatusAccepted,text) +func TextResponse(r *http.Request, text string) *http.Response { + return NewResponse(r, ContentTypeText, http.StatusAccepted, text) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/signer.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/signer.go new file mode 100644 index 000000000000..f6d99fc7fa79 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/signer.go @@ -0,0 +1,87 @@ +package goproxy + +import ( + "crypto/rsa" + "crypto/sha1" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "runtime" + "sort" + "time" +) + +func hashSorted(lst []string) []byte { + c := make([]string, len(lst)) + copy(c, lst) + sort.Strings(c) + h := sha1.New() + for _, s := range c { + h.Write([]byte(s + ",")) + } + return h.Sum(nil) +} + +func hashSortedBigInt(lst []string) *big.Int { + rv := new(big.Int) + rv.SetBytes(hashSorted(lst)) + return rv +} + +var goproxySignerVersion = ":goroxy1" + +func signHost(ca tls.Certificate, hosts []string) (cert tls.Certificate, err error) { + var x509ca *x509.Certificate + + // Use the provided ca and not the global GoproxyCa for certificate generation. + if x509ca, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { + return + } + start := time.Unix(0, 0) + end, err := time.Parse("2006-01-02", "2049-12-31") + if err != nil { + panic(err) + } + hash := hashSorted(append(hosts, goproxySignerVersion, ":"+runtime.Version())) + serial := new(big.Int) + serial.SetBytes(hash) + template := x509.Certificate{ + // TODO(elazar): instead of this ugly hack, just encode the certificate and hash the binary form. + SerialNumber: serial, + Issuer: x509ca.Subject, + Subject: pkix.Name{ + Organization: []string{"GoProxy untrusted MITM proxy Inc"}, + }, + NotBefore: start, + NotAfter: end, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + var csprng CounterEncryptorRand + if csprng, err = NewCounterEncryptorRandFromKey(ca.PrivateKey, hash); err != nil { + return + } + var certpriv *rsa.PrivateKey + if certpriv, err = rsa.GenerateKey(&csprng, 1024); err != nil { + return + } + var derBytes []byte + if derBytes, err = x509.CreateCertificate(&csprng, &template, x509ca, &certpriv.PublicKey, ca.PrivateKey); err != nil { + return + } + return tls.Certificate{ + Certificate: [][]byte{derBytes, ca.Certificate[0]}, + PrivateKey: certpriv, + }, nil +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/signer_test.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/signer_test.go new file mode 100644 index 000000000000..d0e24d298513 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/signer_test.go @@ -0,0 +1,87 @@ +package goproxy + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func orFatal(msg string, err error, t *testing.T) { + if err != nil { + t.Fatal(msg, err) + } +} + +type ConstantHanlder string + +func (h ConstantHanlder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(h)) +} + +func getBrowser(args []string) string { + for i, arg := range args { + if arg == "-browser" && i+1 < len(arg) { + return args[i+1] + } + if strings.HasPrefix(arg, "-browser=") { + return arg[len("-browser="):] + } + } + return "" +} + +func TestSingerTls(t *testing.T) { + cert, err := signHost(GoproxyCa, []string{"example.com", "1.1.1.1", "localhost"}) + orFatal("singHost", err, t) + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + orFatal("ParseCertificate", err, t) + expected := "key verifies with Go" + server := httptest.NewUnstartedServer(ConstantHanlder(expected)) + defer server.Close() + server.TLS = &tls.Config{Certificates: []tls.Certificate{cert, GoproxyCa}} + server.TLS.BuildNameToCertificate() + server.StartTLS() + certpool := x509.NewCertPool() + certpool.AddCert(GoproxyCa.Leaf) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: certpool}, + } + asLocalhost := strings.Replace(server.URL, "127.0.0.1", "localhost", -1) + req, err := http.NewRequest("GET", asLocalhost, nil) + orFatal("NewRequest", err, t) + resp, err := tr.RoundTrip(req) + orFatal("RoundTrip", err, t) + txt, err := ioutil.ReadAll(resp.Body) + orFatal("ioutil.ReadAll", err, t) + if string(txt) != expected { + t.Errorf("Expected '%s' got '%s'", expected, string(txt)) + } + browser := getBrowser(os.Args) + if browser != "" { + exec.Command(browser, asLocalhost).Run() + time.Sleep(10 * time.Second) + } +} + +func TestSingerX509(t *testing.T) { + cert, err := signHost(GoproxyCa, []string{"example.com", "1.1.1.1", "localhost"}) + orFatal("singHost", err, t) + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + orFatal("ParseCertificate", err, t) + certpool := x509.NewCertPool() + certpool.AddCert(GoproxyCa.Leaf) + orFatal("VerifyHostname", cert.Leaf.VerifyHostname("example.com"), t) + orFatal("CheckSignatureFrom", cert.Leaf.CheckSignatureFrom(GoproxyCa.Leaf), t) + _, err = cert.Leaf.Verify(x509.VerifyOptions{ + DNSName: "example.com", + Roots: certpool, + }) + orFatal("Verify", err, t) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/test_data/baby.jpg b/Godeps/_workspace/src/github.com/elazarl/goproxy/test_data/baby.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c377bb8e3246026430751ffc9ae76de178435a09 GIT binary patch literal 2571 zcmbV}c{J4PAIHCAn6Yb=H5ploD2zQz;WFg547rvmldMC?F2={eJJ~Jn!>)&hvSm^M0KVXNofm95`ofYYl)v z0081HfU^e}TiRQhU%qI6TEWNjq~A?1AB70*Q(6kD;i1?-9|hZUNQHBktdI&3hFWSG z3c=y-H?U#16igM2^aME+z!KmA?Q7q8_Qea{cRpTTFfW7;0{O9_0{jrD02Bh@7vdKX z+!wbJ77`TRKkR?<&r=>gFqlse3W5G;`Jar_4Z!#T1b_g86agL>2n+*p`T;l>lJ7@6 zF6lo4@ql^xAW&`$A#Ov<0d9OSm>Zgpo0!`j%e@D9VSM68wap<1QSMO1;6sSS+z0$h z7EL`8_M;4Co$Db<0)moK(lW9t$BwJ2ozT_OH!w6hZTXXxHPXiRoWo_uU!0s>Ts=I! zynTGJexYIE5s^{1qLXi@+)4d4Ej=&4ps=X;-ftz9_^RrMgqqsAC(Tb=o{?IgxApe* z4-5{y9R6c${O!cM$*E~Fl{PoOu(I3xp?+x z0Nv*T@kDYr7{<$YR2w30j)J-eA5=sn@*lFuebCe+prm8ZkhmT)Dk!O}OI2a+qx~TJ z@4%A&FS37u{hNyn2!lb~;elbm8DQJu%@8)=MXR7_$Z4nUuBXt+<4x7lT257rsq zP49az`5Nc2qw9BCv;j}wHjz5|rSe(EYxmKr6GA+(A^uBRQPJ6mdXdcwlO*)^O71sl zFNm#Qyr5ao9pyF4+ndjoN* zk94zm(d+$RG0Hmo%?0GycdOQwp+F2S(J?Z9Ob+;Cc4ku5UMjq^D ztKa4TNk!~vM(O}2rSnkJv+?6kI!JAi?T_0W3Jou5O##Wnb7L)ie;QTiq?(k`+X*od z(fG+B>blXwSXku)dwsvjoeFE*0wa}nmW5;NehqSJ@?ZE+f4VmxT;C&(vJ&|U|T=1nB1O2ub*gb=vz;IM+WAJzq(zq@v z87erXhh9al$CN|FtDcvgBrL}jJW5cN39Oi@tW%l4kx0r$hFamA9v83UW145X$UePx zOr_a7U9-~|-;O8b*OSuMrrFvRy`Ac;!?PdL+a?DpL~(kfI0JuwnRe$djY@70fIo%9 zhoMCCZ^RD0A-e`Mn`75J@(_*uvq?s~>Puo`tYSx2?A(3LON>lE@^2Lop9007s^(PE zKyh7jnBPYoZhJ{n9IJN#NC=M&z$TcH|6FhjKJn=n$>CdOq`^J zhWIA#M44I}tytkwchGEI)^AjmF%2_Cf5wNID{hbToZVrAsfpc3o)h1gBKzU64tM!f zCVZgWcr0kmJ>%tbNnU7AgqMqc*{y^Fgvr3r5jn!D%Z>>ae#A`1O%JbpI;B#m)Gj$W ziLeNz@K*G;bUP7co$8fWW2SF?{~VP2#arg8U>&Nmq}Y2qh-4jwSuNe+#mdVvrIJ@W zrLXh|BYev*JP21mkU6UQ>UM&)jRtb8dG9E)KEB7e^Ay{dxt2J0=R==UlTa<0PR`6n zFRR|sM-+wJ9C}3#UQbE|z$2qY7}-u~wh0SGZC?DTO?IQc zZ~}8hIlJ`bU;guF$Q%7qT~9v_qdZT{6+I%Q^8@AUrFpj>Dh;&O7X&DIga*zyMZClKBS6sK?@#j2!f1mg3TF$lZ!E)>m zgm$(OOXuNh0`b0wZl3j7osN;f}3NXxjH9SmEY znYyY?yQ+yAw{_MvHGOrrxbH@uBSpSyi3G4qN3R+L(ec=M655nSTgrs zqD{v;@0ldST$|4ov?Ci8SXK&S@-hRHrrQwH*Cnhcq<;B{UF^AlNSkBtXx@f5iCL{f zmiH&7afey9JJKx6L4$2_6x@aJp{dcL(`)m7y2{pyDx@)&6(#r_3p_=;*)zTbyB!6# za$8|zgP9hEzv?;@Oqe-#Xx;2;s$9hIfCQDIvF7lF10+ZhIDpxiulW9W-vxBk{CL1y zmF8s_p7MJy;(J9{&1{a}0vl9K&tt`7p66dktg>shuz6GKCC3motW4iMeqaXA0Rs0@ zmsmm63VW7jZ=58Fk&~>4ju*kw!*3>Bxl&yspduru11e(suUv#VPrI;;ZOX^~QmxWH zA(gE~@EwF8gZxy9h})$DFXG0HgW)|GCCeWDU!qHfu{C6?;Dg1e~;JW^?v^Se!U;B$K#!BW}-_=b&CoB09t)LEsKBD_|Ft%|7N9Ez~Vnp`s>*~ z1psQM|4anr6|e&Um8lmPY-Z*O3xYlMg!%L6gTXxhfiMrRC++|cGLNu?T3PO}suC7Y zK!!4yWQkn08z*kH3iUZoZl_Um3U3R3(9pMii#uZ4$6FB zND;*hXoRJMN&yWWq9{a;fF)2w0yq!5xvl|bVt}*Y%greu3UQVjP6XJ$|r z+>^8bPi3Hb>|u&F0FeZkJq&vlfmty?Qs2fy52$Mfy0Hw@^#BD8AZeZye;*(T2b>4^ z_(FiVT!2~U$VTOt2$Eq-_+L`F^(w9WirUfk6k-A7Ha6n7g|PZ;BFyqGH(c^G#L=O- z9MaLU^vlO108o_4@^9PY%MdIb5{s2jZ=!P&+x<#*eb?1>?P_DJ%KzWtR)QiYu0$mo zZbd1RMfqM8-#Z}jd_-OJjF9S4Pp{Dk6z$C09Q*&5jrNDvt#fl5>+7=yy&!v+A)AOR zDB7vp=JLs(NR2lX9f3&n?v@XfnGJ531sDx|m4P~4weltV&qM7>of_v(2ufA(5 zM3{NaoG~C>D=+O>y5w#$=*|7Udz06XSC>T3iPe_mfI4qAqVPYGUmiVT@E!%DGnULjACyusm zp;Psz|La0gh!X0ErXm!%IQg2 zq*(D->v12c2HsFibkJ$bR2~4NV=fXpbfw-#*PE+;qiF=UmMh$f((hpq7{Px|=`HE_RV_clAZOz#s(aZC^B^8-=JEVtp$cdKctFt_R+Rui`; zpSq2G4klrY*6ya1CkY1exZKv%gBU?fX7$Q>xcFsn&QmH-633aN1@C5+8-5XJyZNV= z-BnsD5uzu`H^Z30B*74#1nyPfXG{i53%0(#q5t`l)gXs*)C&8?-R@gxQ`YrzNyP zxjeBFS`M>P{}gF^nM`KeW7J~`{$yVC*=elqA?W@GM%cSK#yL+>7VJ z?Kzj+?WwAj6(1DAG6pg>4Hn05J=Dq9asti@f=np4gv#K@ktvJJhxb?SbC${)wMw}= z_*7=YvmsS_RpwQ?d!6#g{W3&pr*^yTc{H77zk#5Gpe{56ON!XCoU>dgqbg%HR#iGh zidXuSaDHgl3jXR>c92up^(d;)2v)Wj+PU?YD?;MZ>eBab*{LoKgsg#vpA1KXWmoBw z3xKI`OZ4M@nS&jBG{7Y;?_A(3aha_XDj%nHUg^-Md0`yZ_}6?fzL<{!WCFYbhQ_afzDsZAZ{>f_Uut%?3$YVzDsO6W*RNlhO!VQ?xCCSlc-^=+q!;?#;Ba)=BV44CBq?`e48R% zpssAt=00T4K~`s0L|eqGOI32(#5uYuZaePCr(Xh@l^!ZB=3xsC)Fb*%vUen)kTHg_ zS(&vars4a&UI@$lDP1TVwDoE1aZ+lsVa%{NTunTm6SFYTxo!}28sNARHhGH3Z^%dQ zm*Z*=~r+)uV(r2kU(iChO4Q<@wlW>#Bw5j=C z*IW`@NpLx2Q~RwJ_dGfwWZ>5|$LNh27a7;lrHp6;InxiujLmkXZ2gP$&G{ng6WTFi$^nJU(b8ORB% zRf>D7%qz6Xwy9&pJ#0s&Aqa^66GJXVabcvq-;=JTi~ZxoRm3XwS~y#d#DJrJ=m#wN zB)*KbLYz@lDOhP?pPCH9<*aGKU$c6Iv$&`q0w8jZmyzVffHCo75DrC)j&f_bss?;eJ`zG^s0H~7JII9V|3*Gd1 zK3we{Uw_r-m)b0G`yKh9Ps8hKQd=>qCQ#|Nm7UOcVe)#YyN$=!(c~e;?5aYM(fyHz zA>3<)g2VUM`;2JDl{ZZ#35GUe+>p)AYH!7Qm4Hwj)fx+_NWw$1YL%9hCSl)c)2e+N zI5M(RAk5e&`Vnd<2GvO9vqx=BwGkh&hyT9sXbO?AU21gsn}Fa{V@XayGUM3bEH7t; z@F5$ViLjxsx%jtVDyn*0ACGxVdhBd2iWS<^Wmm#C;dFkqbMs$lmmYrEKmDzZlYoy3 z;o4)Hz^xy@stmknY^mDR02S)Ew@wUR%WTxfLpe&#oEz!~20TlU8^&c1_isW#Y! zvnH}Op7xk_Ki5BRAgCjgD|gPa&wU+hOnd|O`-vxsFUk4nn!dMwo93QWwX(>V5!utF z>%Y@=g*Xfrqm`$%epP^3<@47HUEd*4E)j0iD$PGn3$I=wpSts$8_F%=4az&p)5*8Y zJGXbWZ!;ZHo0!>M47BP2pQ^ z*d=dz*kKz((8{Ftb?TOKhcedTs3qvA9=E|{(_)iXtK{Un8?>u(l{IDB-(Jw3hirB5 zX*~$*Yz+4x>~Iz;U>8n0Lw~Ve&rBC8sxEXMhrtMagwrL~g~&j`OYCs_T(IClO^0cR z(}~8u<{Wq-aa!H}3O{>@f9c&FTy#8v*Cmj}$BtcPo@UU4Ni%XYh7)KL_6i@UDJrh2 z2VU-6l!3<{j|mjAT@_yqBiUFbBG0DxzGEkHCO*KA;8|aNUzA+#cJFc2;xK5;?XJNP z8L3An56|~e!T4bA!;ee!rt2?wUJ6}%{`q`rIbL?2#gUbDO+uV=eBmY|hWqz#^0?_) z7z04qJphP`1%N--|F{DH&m;k0*BJnmasYr0_Qt7C2LOnx^tCjtLgs(qY?HXGnaP%6 z9`i=*P)0cUO*tegAgR2Cl#N*%MJ`t^)aCVBNVxoUS&?%ug_{0dZ!J89I74%y3Pnsg ziTRzh7LwxRJR=9Q%pwz`(k!n#o6HXRgF&2?RqgwKS0<;_<(l^{tO@w*0^~u}w09WC zWf)fp@c&Xx>55Tkh}ngj8zX(WE$nR0H`8gmIE+@w zpI~&)fcFVz+^L!l(Pcm7ycIJJe{vx)9}vuElp#FGhc!i?Su&QtJSU4JD?}l05ZB&m z{}m|OfUe{5>r?fm>4-FPQPoLhSjN&(bwpN+qRD^kEo(W9+DB=Q?6|(SM2(}U{ORPE z>%>`|P1X}cuB!mUpW>`;FgEHsHn#v?xV%XBeh4k!S%XG9Ubsv-8vIlAp)BJvU3H%Z zsu5%kVJY?1Awu?KshR<5%xr(1Zloo&?o4q)UQx>skrhYfzMoomT0Rx>l6gYu=Yx*8 zgWrt9ao|hi&Pid*Sp~s~jqJPqjV-eMgYPl4A%ZJ2mPwJ?yasoE0cvT z7~1}GRWXL;K7%JxcQ$9OTUoYNi`(W`9~8j-;cI`kMcpA>_zFse1U7$HEDEMCakaB@ zeykU>&owH5@@_gM2yVLPE}{ZEY@Db&V#33D&Jug5#vrsvjOyU|HBKf-1X@KVQ=RO) zeK^p)APN#Qnr1%0#Ew3Xd-O_v5-L_-r0pU}Rd+AMwUk)6;o?}wp`@X``<2KLZ*U+IBfRsybQWXdwQbLP>6pAqfylh#{fb6;Z(vMMVS~$e@S=j*ftg6;Uhf59&ghTmgWD0l;*T zI709Y^p6lP1rIRMx#05C~cW=H_Aw*bJ-5DT&Z2n+x)QHX^p z00esgV8|mQcmRZ%02D^@S3L16t`O%c004NIvOKvYIYoh62rY33S640`D9%Y2D-rV&neh&#Q1i z007~1e$oCcFS8neI|hJl{-P!B1ZZ9hpmq0)X0i`JwE&>$+E?>%_LC6RbVIkUx0b+_+BaR3cnT7Zv!AJxW zizFb)h!jyGOOZ85F;a?DAXP{m@;!0_IfqH8(HlgRxt7s3}k3K`kFu>>-2Q$QMFfPW!La{h336o>X zu_CMttHv6zR;&ZNiS=X8v3CR#fknUxHUxJ0uoBa_M6WNWeqIg~6QE69c9o#eyhGvpiOA@W-aonk<7r1(?fC{oI5N*U!4 zfg=2N-7=cNnjjOr{yriy6mMFgG#l znCF=fnQv8CDz++o6_Lscl}eQ+l^ZHARH>?_s@|##Rr6KLRFA1%Q+=*RRWnoLsR`7U zt5vFIcfW3@?wFpwUVxrVZ>QdQz32KIeJ}k~{cZZE^+ya? z2D1z#2HOnI7(B%_ac?{wFUQ;QQA1tBKtrWrm0_3Rgps+?Jfqb{jYbcQX~taRB;#$y zZN{S}1|}gUOHJxc?wV3fxuz+mJ4`!F$IZ;mqRrNsHJd##*D~ju=bP7?-?v~|cv>vB zsJ6IeNwVZxrdjT`yl#bBIa#GxRa#xMMy;K#CDyyGyQdMSxlWT#tDe?p!?5wT$+oGt z8L;Kp2HUQ-ZMJ=3XJQv;x5ci*?vuTfeY$;({XGW_huIFR9a(?@3)XSs8O^N5RyOM=TTmp(3=8^+zpz2r)C z^>JO{deZfso3oq3?Wo(Y?l$ge?uXo;%ru`Vo>?<<(8I_>;8Eq#KMS9gFl*neeosSB zfoHYnBQIkwkyowPu(zdms`p{<7e4kra-ZWq<2*OsGTvEV%s0Td$hXT+!*8Bnh2KMe zBmZRodjHV?r+_5^X9J0WL4jKW`}lf%A-|44I@@LTvf1rHjG(ze6+w@Jt%Bvjts!X0 z?2xS?_ve_-kiKB_KiJlZ$9G`c^=E@oNG)mWWaNo-3TIW8)$Hg0Ub-~8?KhvJ>$ z3*&nim@mj(aCxE5!t{lw7O5^0EIO7zOo&c6l<+|iDySBWCGrz@C5{St!X3hAA}`T4 z(TLbXTq+(;@<=L8dXnssyft|w#WSTW<++3>sgS%(4NTpeI-VAqb|7ssJvzNHgOZVu zaYCvgO_R1~>SyL=cFU|~g|hy|Zi}}s9+d~lYqOB71z9Z$wnC=pR9Yz4DhIM>Wmjgu z&56o6maCpC&F##y%G;1PobR9i?GnNg;gYtchD%p19a!eQtZF&3JaKv33gZ<8D~47E ztUS1iwkmDaPpj=$m#%)jCVEY4fnLGNg2A-`YwHVD3gv};>)hAvT~AmqS>Lr``i7kw zJ{5_It`yrBmlc25DBO7E8;5VoznR>Ww5hAaxn$2~(q`%A-YuS64wkBy=9dm`4cXeX z4c}I@?e+FW+b@^RDBHV(wnMq2zdX3SWv9u`%{xC-q*U}&`cyXV(%rRT*Z6MH?i+i& z_B8C(+grT%{XWUQ+f@NoP1R=AW&26{v-dx)iK^-Nmiuj8txj!m?Z*Ss1N{dh4z}01 z)YTo*JycSU)+_5r4#yw9{+;i4Ee$peRgIj+;v;ZGdF1K$3E%e~4LaI(jC-u%2h$&R z9cLXcYC@Xwnns&bn)_Q~Te?roKGD|d-g^8;+aC{{G(1^(O7m37Y1-+6)01cN&y1aw zoqc{T`P^XJqPBbIW6s}d4{z_f5Om?vMgNQEJG?v2T=KYd^0M3I6IZxbny)%vZR&LD zJpPl@Psh8QyPB@KTx+@RdcC!KX7}kEo;S|j^u2lU7XQ}Oo;f|;z4Ll+_r>@1-xl3| zawq-H%e&ckC+@AhPrP6BKT#_XdT7&;F71j}Joy zkC~6lh7E@6o;W@^IpRNZ{ptLtL(gQ-CY~4mqW;US7Zxvm_|@yz&e53Bp_lTPlfP|z zrTyx_>lv@x#=^!PzR7qqF<$gm`|ZJZ+;<)Cqu&ot2z=0000WV@Og>004R=004l4008;_004mL004C`008P>0026e000+nl3&F} z001EcNkl4UG(%o-gX2m))0EmSPKmsJeMIyz`q!x>oC|TtsmgOIc z?NmiBDRLaAY*#9FoJxK;sfsIJ%96EMCCRj8izO|1o_o%B&i6GjGyZ>mks$dq`9qE^)~sF&-}ezlJq!n9b^t5y7K#0KT~glz|3&Y0RY|@T^G(-YhkQImSxKh zu+}0;5+q3iV+<;l%EmRT*L-Sfa`G=FVsgIut1(@i(e zRLT`3nYn%2)~&x{t()kDA$s$37#())KD;CRYer^tpW(!KGNBw;8EvfWvqvI>?%K5p;T(Ka6Ab?g1 zju}cRWLbvmuD!0axVV^{I(4e{(n~MN6DN-2nO(c=xpU_TMAT}vbJy><;Zys*weO!j z*ZW2V9vt}pZ3HlwMn*?}d-LXvzrB6?HowtmU~#dDiHSA5X6+h!{`oy0P*S~b<;s=b z`1lHy&*veP0$qq*1{^yO3CMs-K`vJ!A+UC00?nl*w`KEYdi;k!qn>TF^)@|6}%H;}cPm@v#Qp*0?u>&(eL}15=lNh68V@Q)2LEzKThJp@5OaU5gYwym<=X`|U} z;>gis>D>9**Ibyw;mXT^l3HB$-Y3B*77G;t4%(eAWyYcxhUkSM?z!t8eCk&}g)LWK zP3$Z@Kj^PA00_RgGz>~e-xw(+q?CMNk4Py&0CsHOfu*GezWFzQ^NNt-iE6b9khpRO z?7Z=o%PVX`5Q5JJEj11Tj&jY)6axznv&G435bdbDun^y#H5PT&C4{r5g_Sr-Z+5XBK5 z`QG0@-R*YwwL0zF)~%bOyY9Xl4}SPTjE}DXh@qrfcGx=x)gT&Z@MWbmgzP(^SS+cr z@iF7rZJwW-yJM-4|C(0%${jFs?&M{n9s)ay+itwUn^?bIcf$~C)=uEY9XFv+D8M?4 zAPC-Q*t^|l?3`nvl#oh^O0DK~dp!(~jNI;N^)(@W23z3#`Lmao0-S@Cl0N<~{^h@& zT0eEO$ui`NB^2^SF!#gug28WLID^>%xR4D&M10XXOGHwTFhry>V}u|=K3_n!R(+2k z3~Ns>T(JXmu6S7&nk>WQx^){@uU+#?+SBlSjY1(0-}3+hlzPV=!-0sI8LuM(ur-D< zV}$2^w9ilPuEGkE-;9>>Ff`?ttUhQI&LBl_2W{nzx49oy0C z^`MmIYOO|{PG`$2FTV&SCEf}R5djxK*IPOu>~=0E|G_y|wOO_~O=7583H=Ih;ROK* zVi3?+j_)&PPUG9({(HRm;){sF5Y=iGpZLTluyf~55HY&F9;8$_eE0~y|Bv6t=;#Q- zFhUrH_~tjhfnu?M$;nAbB_X9ksZ=TlegGlpCmcZ3uTsu2PM$i2*}1v5bdowhGjn+; z2pB4ss&!_-S_3~Q^fA44@VtIB0HC8sj^G<#|0-VFcNj;H9)s2jNfM)0AHtvi#b2OO zt)f~kqZ{_1mB!rcd1$Ti)YDI4YTY_?J6)8^WqkcBe~wT8o6n+Jt&=qdX_AuG`X^SG zsNa$sYw_eWKf<9yN8Wx5T(xoIWo-d#ZEva3%B*t$2vT}*&cd2L*b!0!1U&hJ@8U;K z{xgJM!}<;DvADPZV+}g3Hs1fh`w;{lM#e_5CJ4aHn3|l#`t=)d`}4QsyWjgRW_vw2 zb~u0LFdB2`P^%3ii4%lTXcm_iATEHRLCTR*V9)a}qSftSeqsKiJASJ&I5T}&0l-8y z7v!0pgE0(~S(ZvbDnaZ3b_|k)HEY)*?^#$SP%f3Qckc^WF}?zy`*)wk-S^&$N~!z? z2nbwr&DEHkn85ZMuE&?Y{8gMd@-o&>PGIV)Yp}GqfNr-7cHDN%fbDk#3_vS|*|`Or zn>mkSsnBn;)9md5U6Q7F*GHBjO%jo2u@OXw;s{|B!Wh?wx`KcKM!E5pJF$85)!<|S z=V#}!ZfYGq{pn9*=PkEhvM%i0B^%5!V{B|3AN=44FjOnyLm&AiRI7Egnk|G;h<2-W z1QrmEARwWI#&BsR($K+J`%{B1wKzY2xs!lwXm}*{eIG(e6!Imq*1?(>LP&@~qf)C6 z@7D>KXn=0f8gL{*vNXz&l&9HaxZdhw^&2`rT06aej zV@yBHwT~$5O?yfnS-o;4qBy~UBYROR4Gk*7yTk$i>wo|0W!TSf&bjNiZCm*FpZ_d^ zTprBCN-Gj1L4sh+>u}gP$5JZ3>Gq!!NgN@~QYftv#}Pc=!@k#E!(aW?mtc%R6vx>6 z!X89Xg0Ft{%P1BLNYnnX)ESF*s|6v%frUnEI*MbgU9)C6`Mo}4co$8bQVQ?(@H`Dc zw0ie*&)xX&x4vy!%?4Vn2D+UdOqRiB2G$sGAL}y_IcrVQ?Y5z{g6Dbt3GyI6|K%6H zfQKJ`82b<|P9^xU2!VlJf3Sx%8X!^Yd>RY`Axkst-n|=#4j;m))2GmGcko-k^?BTK>n%8Q_6(vhLb+VQ;=&@H zeCi4J2JLt5m8=2)I(IOvO@(P#|dS2_j;xJx3%uQV17C;Yh#FmvGLa zvDCnQ_uYq0n>XU=r=G?azVJnS<~Kishd%ld96NR#OG`@_9v;E@nHiisc>+&7`J|hk zo?g-Ic2=D}ed@MKwf?V1N5^(;*}Nr5({!!&^j0Bd0EQ|S3QwOnapKd~@yWMxf%JUu za)u#wDb;@zgf&7+Ge19XXXj?k#M-qk?+2pS>j@epMFz+@C!94BB(kN2g&>y;gdYT8 z2QWKiS&CIFS7FQ5SL4MOU&8Rv5ZdiFjvqe(&+|~P)^O&`DLnpz$FY0&^Tt|ZiO5># zqGq#I7^>H{+jjSq5hvqNwM`QIwCu80}7*@`b_x z*Gu+14@*l8dgPHuaP^kW7#$wQ@#Du41UZZjkKpX|Sv>jF)A-Ske#D(_+d2jrYn@iw zd0KO+RFVwXD2fb`a4T1>qOI3$1O;=V>mLOQ_dt6vr{o&o7jhmKGkj)}Fj#2S_P5)oazS_`WA=wTjc)Q=ZnMR4nRT zE|3gRv(>`B{RhxkYNA*w4Zux?FbttRA8DE)47+Hx8tCQLhi9G*rbVyZLoy=tBK`yYf*km;aUT`E~8?Oks@C@PS&rj+3WO!}mPrd496E zu;`ktmSE?kvkoRR+S9L3-~oU&|GK5hClV&IuO0X8lGK;zSdCZ=l#Vaqrf(;usU}$Ivp7u~5s$pbw7`~^Wm4|k_ zgGRFfAth$#=5YAXVPt8NnJh~yrIJmJ5hR4lNDJ4u|VXPGg4(!K+4?ZYn zXJ&b3c2+xUrIb=cQOJ&&8jZ#IbLY-nvY4yaOkildela&lU^p-n#DI`cN`VHLo(Sut z5bHoll$0u8$RQU5C>Ba!cJO^4*7k`J&(ml&+lZ4Cj6U7<>Z`B9^F4T;h7__tR2lRG zwQ3E$u*1$+5=t?F!l8O4D;Ip*iBcEESzc?U**T|#v~%-w5Q3amS{P%65Yj&S=%a!i z7e+@%z04R`>!gxG8e<}@^s$SL-!M9YpS$x8$RpmHk)sS_ER3_*SV1!mu57i4nwOS)TmqQpC5ydgmG?VpuRVGPdk~9U3Q2>eg zg#`y@_@1X6JE5eqN-6R@Pt|I*e_B{vI9RP#;VFS@3mUg?+>XM^GLi%4?bu<4bA4yz z0}sX-q*=xV9GS77B1|p|wJ<&_$9O)M`T*8g9e)JxBovK~PE~@O(sZ zjB0HdxgfxSgNM)ydss7GLou)2wi|Ac*KOVImC9ADShEh5av8>0oIQIQomLwM_Prt` zk$(P#kgaj8FpBa*$edOR);Tm58#0&A$sh>e9M63I^S|{wAN$3RC7$+h_Sk-W=X0ON zLI@<8L5Y6~dOXciWF|wUQc1HcON})JV+@lJh~gMwuZMg-4`PNj4uyOHo_5IRix?Rm z=?^BQKrWXXpbvpKju6EWmX?;VY10M(giJTOymoT!^r_<~+F`F(Q%WJ9&kq_SA0!A; z2zZ`CzF2}*9(uhVtTia)3J^*`YlU*T0%J|T#Ldh=DT!{khtsD|<0Bvacz)&TwLxDj zkwR+^&Ph0Cm^AH2kdy#BTyxzG_`rugY+ic#l{2Fw!#^A!U-83do_U5^aZC5}btNg} z3w-yzcVl{H2EX$keg}t-p2P=NXSi(}een>JCcUWb%wSx+PfGe2fVW(<_lVCOJBJ>7fqsi#C3hT-k+xpQg#)MU38hPK&i z8s{t~rY5Vt=ReTtbU(Z}zpw#d732#`o(gP|SsGP7Rr2X25kTChq%QS|C32S7m@ZpM z&-Nlb$Y3W0K`JSXbrdHtq*N#r@~Bi&_}YhY27)AF8N<0jhj^Z0t%ETs+T9MaG{O9A z4?!W1dcB5XseDOKZ>@ze22u(bV<3b;7>0Suz zQc75BFgrVoZ+`P{)ZFa(!1pzETb&gPOA9}*l$u#wTujHt#=bZbvH zhP8Gelu!U+0MHzaF-VdWXV0GH1N-+2Nkmde=)7`^3-iMZ^9%p2P%Iy>R?3q~DO1el zy}R$Y6I-v}hWFil7Z&E{@x?Fv)ryk`5C3ANHu2w8qXY#9LJWfM6WEYf@nA|8TsH&< zI&(4;NiU*+ZKVqDDxn0FlrqTY0Cs7b8Vj%=M+tR1Ag>iSiXZ9xtT;g%q{Cu9<49&AD~^TvfQhNTdFgb1+mJkp-WIp1dpK{hk!g+27Q zM~W;n=$H&rNR)~tTz$qlX_`Xo zzKD{+{aztSaq!>)a?UBK1f-AzJkS~A; zXg2g_m4X!>tn|PpfU^ZS@~w5NowGg>K@gEp zf&>ZjJdZukMFH^F^r43!rNXn%K8rv4qd&qQ z{NW$s_kaKQuzmaXL1wpLc92ppccv!GptQ!o;c(<&jm428M-askND%t8oS`I1APWkT z2txGrhWUIRndwill+*}<914X3&P|^~H|}+6Vaijy8bSbq93)amnULU!1Z9w5>w`D| zTpo_A1ga>d91plFYb+aM!OVi4qfWPrPOpQjHeH2J{_-#5#v5TU%j$1-&N;?FO~%Y{+!thoy&jGpIf5vPAV?0X zg+iZ^vT*E>Bq>M`(kx>DuqFct%c6}_XHG-K>cHe^8RgZ21P^#s0tB;>WStU39zY(z zJ^`K}@<@P8nzfT8L7Ju@5@fBVEX%mt>tShW5x4BziTm!mcfb}4kRVJ>Od<*+2qD4N zVSH=^zx>HhA`BxmmKqR3Ku8JgdFXF<6Ac^-=6+&~q7dC~7l#fX!jJdvh1MEM_jSg` zWH82{(QKmK>A)C+IEu4ww-+xiEuq`#LMjOwl-JW|&Y?8S2cqx*%IS+xjDd(j07Oo( z6v9CWI0#~rfRhXs#9+s~kY$DeNlHm#hDefxI~_9lLSEl`+fID==Rb^-r_LZV2E}3- zrE(dqW)nmN&-0MP8EVxE#>dC8xVVTFD^?&)(>_l~B+}FrvNT1KB*+&Fu*M>cB1F9& zc0Icbr%s(3Xb<|Ko~9`n1lQjVs8(WQjP4SkJDo1lEFDy2!kN>jar*S}Q)|NNGv9yW zC6uq4gc|5L7>?LU#V}wf0tgOl4cHj~2?J55+c_2l!SzIc>cxVJO)910_sv>f^epXK67>*gWTCLwd2mxk?IF8Zlh44L%G|NZ`fp)8fPN#!> zuYfQL(P}ht@W4U$=%bHP81*O@1c;LuT4_irU~L~K#7Tn1r3O{2)xfdS{hnb4lao{M zJP${Y9z(R$_=iXKADDaM#A#Z;E`Z8;J_wn$&=$&Bw(JaB$Br#IwhSa7>Jg*O&dsK2 zI<1u|XV#F^9!a6VY$=Ijw{PD;+3R&tAFAX0%rw$CMX^}K==c~&2xz4+K0Xd1h=m{$ zBnNz8gvG_hK3l|$>FIL-_gA;sYGd!7J=nc#mpFCi6r_|e)qFpb%{P0UZ-b;RU%nuLl%@kV;L2L~t89N9#2xdqy zv;$HiqyWbZbOAJ5t>#Mr#`5`G%^E`_gmBhT7=~{5?&rk5ef!UmuMyu6!Gc)ItLx&FK(=5f}!XlcjCc51Y+N};wojL)}^WEI+Z2RQt)AI)p z9-MBqnz?)~x5ij5Tk8VP^Q0UIq@C)t=4l&srL#& z1+5kLA9qQ4es0b_|NQgVxN(!s<@3a3oOM=e-x~-k6#`#jeqm1j_{Tp!^4MdK?OL^J z)oKC)=a?GJhK-{rjN|xxtKFUp!>|j$f*^$uUaQ;93l?R7OH#>E&-02>Nnc4-CL&J? z6_6)%yWR4#AXk=B(krjLietzAAUl2L^#5tKTHh8>1to%Jo(idy&MGPr(7jNI1}%|; zz@QE3llX+bq7#w@v~$jHEG-r5LqoSZ>m>GX`N8i8M0=TXxJ~$F`sFtkFrD zN~MHmP~e>N*|8E3c_BnqX*uNiUfmhj?S;KF%zS`I9ChGE#wP3_G8C~>Fas6CG9f?) zP!Zsagfk$d3{HrVGLR4`W`WpPn|3-KDijJkl@iJrlLkSK4jnv{J@@<`{lO2sAAZoE z6chTAnL(TDI^C{_qiBD*RzAy&JaLXQKeK6)+B8g@OPsS=U+Th+(VtVmUBv437lc_l zX8|S<^%v2xWoIoJn<0*4Ggwpuf?;+U2##T4IR+R|95XJa33Q5g4j^?s7o37R>_YkNa#}U{$JoeZ>+1qZr6(hsLQg{L+1WcA)7St+Fj!2&Q~ z7yu4D_#JSU{H(#kXoE}{96EJIML4leU%3c z0T~DxfMoF15I|di&t=vec9tV0)v7p73a`HUvH%ec4~oi#K8jx@h5-sC;0yN zzyDOD(fAI;Kwj#W!_*27EFm2UIARdEbMN#+JD!}q66%>4Mqe|1m+BY5gII@;H+EL7Y))AU39kt-fl3# zT2Fw>11G{ueDy|Y>l|VWqiPtsqM+*rffZOZgp)ssZQ*K!5bc zZz62vw^O~o~cUVIcs)f=&OaKLSe- zi9ry6m-Th8TmG&eHF-C(-^)3-e4YRneIrRhP)UHSbKD_<&ulXRSnjV20|Z}!NHZG) z7_(f_1`6aiGvJM3OYdMfz2W`aI^#+nx>>HDs_z80U@kE1upp<1oB|;GY3jmVuw1*E zDW3xdPkQt@0EfPtmP^;0CagF9?>Bi*Z}KMgRZ+07*qoM6N<$ Ef{+FK(*OVf literal 0 HcmV?d00001 diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/roundtripper.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/roundtripper.go new file mode 100644 index 000000000000..3651ad86c0c8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/roundtripper.go @@ -0,0 +1,19 @@ +package transport +import "net/http" +type RoundTripper interface { + // RoundTrip executes a single HTTP transaction, returning + // the Response for the request req. RoundTrip should not + // attempt to interpret the response. In particular, + // RoundTrip must return err == nil if it obtained a response, + // regardless of the response's HTTP status code. A non-nil + // err should be reserved for failure to obtain a response. + // Similarly, RoundTrip should not attempt to handle + // higher-level protocol details such as redirects, + // authentication, or cookies. + // + // RoundTrip should not modify the request, except for + // consuming the Body. The request's URL and Header fields + // are guaranteed to be initialized. + RoundTrip(*http.Request) (*http.Response, error) + DetailedRoundTrip(*http.Request) (*RoundTripDetails, *http.Response, error) +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/transport.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/transport.go new file mode 100644 index 000000000000..fc1c82b1a512 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/transport.go @@ -0,0 +1,789 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP client implementation. See RFC 2616. +// +// This is the low-level Transport implementation of RoundTripper. +// The high-level interface is in client.go. + +// This file is DEPRECATED and keep solely for backward compatibility. + +package transport + +import ( + "net/http" + "bufio" + "compress/gzip" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/url" + "os" + "strings" + "sync" +) + +// DefaultTransport is the default implementation of Transport and is +// used by DefaultClient. It establishes a new network connection for +// each call to Do and uses HTTP proxies as directed by the +// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy) +// environment variables. +var DefaultTransport RoundTripper = &Transport{Proxy: ProxyFromEnvironment} + +// DefaultMaxIdleConnsPerHost is the default value of Transport's +// MaxIdleConnsPerHost. +const DefaultMaxIdleConnsPerHost = 2 + +// Transport is an implementation of RoundTripper that supports http, +// https, and http proxies (for either http or https with CONNECT). +// Transport can also cache connections for future re-use. +type Transport struct { + lk sync.Mutex + idleConn map[string][]*persistConn + altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper + + // TODO: tunable on global max cached connections + // TODO: tunable on timeout on cached connections + // TODO: optional pipelining + + // Proxy specifies a function to return a proxy for a given + // Request. If the function returns a non-nil error, the + // request is aborted with the provided error. + // If Proxy is nil or returns a nil *URL, no proxy is used. + Proxy func(*http.Request) (*url.URL, error) + + // Dial specifies the dial function for creating TCP + // connections. + // If Dial is nil, net.Dial is used. + Dial func(net, addr string) (c net.Conn, err error) + + // TLSClientConfig specifies the TLS configuration to use with + // tls.Client. If nil, the default configuration is used. + TLSClientConfig *tls.Config + + DisableKeepAlives bool + DisableCompression bool + + // MaxIdleConnsPerHost, if non-zero, controls the maximum idle + // (keep-alive) to keep to keep per-host. If zero, + // DefaultMaxIdleConnsPerHost is used. + MaxIdleConnsPerHost int +} + +// ProxyFromEnvironment returns the URL of the proxy to use for a +// given request, as indicated by the environment variables +// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy). +// An error is returned if the proxy environment is invalid. +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request. +func ProxyFromEnvironment(req *http.Request) (*url.URL, error) { + proxy := getenvEitherCase("HTTP_PROXY") + if proxy == "" { + return nil, nil + } + if !useProxy(canonicalAddr(req.URL)) { + return nil, nil + } + proxyURL, err := url.Parse(proxy) + if err != nil || proxyURL.Scheme == "" { + if u, err := url.Parse("http://" + proxy); err == nil { + proxyURL = u + err = nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// ProxyURL returns a proxy function (for use in a Transport) +// that always returns the same URL. +func ProxyURL(fixedURL *url.URL) func(*http.Request) (*url.URL, error) { + return func(*http.Request) (*url.URL, error) { + return fixedURL, nil + } +} + +// transportRequest is a wrapper around a *Request that adds +// optional extra headers to write. +type transportRequest struct { + *http.Request // original request, not to be mutated + extra http.Header // extra headers to write, or nil +} + +func (tr *transportRequest) extraHeaders() http.Header { + if tr.extra == nil { + tr.extra = make(http.Header) + } + return tr.extra +} + +type RoundTripDetails struct { + Host string + TCPAddr *net.TCPAddr + IsProxy bool + Error error +} + +func (t *Transport) DetailedRoundTrip(req *http.Request) (details *RoundTripDetails, resp *http.Response, err error) { + if req.URL == nil { + return nil, nil, errors.New("http: nil Request.URL") + } + if req.Header == nil { + return nil, nil, errors.New("http: nil Request.Header") + } + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + t.lk.Lock() + var rt RoundTripper + if t.altProto != nil { + rt = t.altProto[req.URL.Scheme] + } + t.lk.Unlock() + if rt == nil { + return nil, nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + } + return rt.DetailedRoundTrip(req) + } + treq := &transportRequest{Request: req} + cm, err := t.connectMethodForRequest(treq) + if err != nil { + return nil, nil, err + } + + // Get the cached or newly-created connection to either the + // host (for http or https), the http proxy, or the http proxy + // pre-CONNECTed to https server. In any case, we'll be ready + // to send it requests. + pconn, err := t.getConn(cm) + if err != nil { + return nil, nil, err + } + + resp, err = pconn.roundTrip(treq) + return &RoundTripDetails{pconn.host, pconn.ip, pconn.isProxy, err}, resp, err +} + +// RoundTrip implements the RoundTripper interface. +func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { + _, resp, err = t.DetailedRoundTrip(req) + return +} + +// RegisterProtocol registers a new protocol with scheme. +// The Transport will pass requests using the given scheme to rt. +// It is rt's responsibility to simulate HTTP request semantics. +// +// RegisterProtocol can be used by other packages to provide +// implementations of protocol schemes like "ftp" or "file". +func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { + if scheme == "http" || scheme == "https" { + panic("protocol " + scheme + " already registered") + } + t.lk.Lock() + defer t.lk.Unlock() + if t.altProto == nil { + t.altProto = make(map[string]RoundTripper) + } + if _, exists := t.altProto[scheme]; exists { + panic("protocol " + scheme + " already registered") + } + t.altProto[scheme] = rt +} + +// CloseIdleConnections closes any connections which were previously +// connected from previous requests but are now sitting idle in +// a "keep-alive" state. It does not interrupt any connections currently +// in use. +func (t *Transport) CloseIdleConnections() { + t.lk.Lock() + defer t.lk.Unlock() + if t.idleConn == nil { + return + } + for _, conns := range t.idleConn { + for _, pconn := range conns { + pconn.close() + } + } + t.idleConn = make(map[string][]*persistConn) +} + +// +// Private implementation past this point. +// + +func getenvEitherCase(k string) string { + if v := os.Getenv(strings.ToUpper(k)); v != "" { + return v + } + return os.Getenv(strings.ToLower(k)) +} + +func (t *Transport) connectMethodForRequest(treq *transportRequest) (*connectMethod, error) { + cm := &connectMethod{ + targetScheme: treq.URL.Scheme, + targetAddr: canonicalAddr(treq.URL), + } + if t.Proxy != nil { + var err error + cm.proxyURL, err = t.Proxy(treq.Request) + if err != nil { + return nil, err + } + } + return cm, nil +} + +// proxyAuth returns the Proxy-Authorization header to set +// on requests, if applicable. +func (cm *connectMethod) proxyAuth() string { + if cm.proxyURL == nil { + return "" + } + if u := cm.proxyURL.User; u != nil { + return "Basic " + base64.URLEncoding.EncodeToString([]byte(u.String())) + } + return "" +} + +// putIdleConn adds pconn to the list of idle persistent connections awaiting +// a new request. +// If pconn is no longer needed or not in a good state, putIdleConn +// returns false. +func (t *Transport) putIdleConn(pconn *persistConn) bool { + t.lk.Lock() + defer t.lk.Unlock() + if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { + pconn.close() + return false + } + if pconn.isBroken() { + return false + } + key := pconn.cacheKey + max := t.MaxIdleConnsPerHost + if max == 0 { + max = DefaultMaxIdleConnsPerHost + } + if len(t.idleConn[key]) >= max { + pconn.close() + return false + } + t.idleConn[key] = append(t.idleConn[key], pconn) + return true +} + +func (t *Transport) getIdleConn(cm *connectMethod) (pconn *persistConn) { + t.lk.Lock() + defer t.lk.Unlock() + if t.idleConn == nil { + t.idleConn = make(map[string][]*persistConn) + } + key := cm.String() + for { + pconns, ok := t.idleConn[key] + if !ok { + return nil + } + if len(pconns) == 1 { + pconn = pconns[0] + delete(t.idleConn, key) + } else { + // 2 or more cached connections; pop last + // TODO: queue? + pconn = pconns[len(pconns)-1] + t.idleConn[key] = pconns[0 : len(pconns)-1] + } + if !pconn.isBroken() { + return + } + } + return +} + +func (t *Transport) dial(network, addr string) (c net.Conn, raddr string, ip *net.TCPAddr, err error) { + if t.Dial != nil { + ip, err = net.ResolveTCPAddr("tcp", addr) + if err!=nil { + return + } + c, err = t.Dial(network, addr) + raddr = addr + return + } + addri, err := net.ResolveTCPAddr("tcp", addr) + if err!=nil { + return + } + c, err = net.DialTCP("tcp", nil, addri) + raddr = addr + ip = addri + return +} + +// getConn dials and creates a new persistConn to the target as +// specified in the connectMethod. This includes doing a proxy CONNECT +// and/or setting up TLS. If this doesn't return an error, the persistConn +// is ready to write requests to. +func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) { + if pc := t.getIdleConn(cm); pc != nil { + return pc, nil + } + + conn, raddr, ip, err := t.dial("tcp", cm.addr()) + if err != nil { + if cm.proxyURL != nil { + err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) + } + return nil, err + } + + pa := cm.proxyAuth() + + pconn := &persistConn{ + t: t, + cacheKey: cm.String(), + conn: conn, + reqch: make(chan requestAndChan, 50), + host: raddr, + ip: ip, + } + + switch { + case cm.proxyURL == nil: + // Do nothing. + case cm.targetScheme == "http": + pconn.isProxy = true + if pa != "" { + pconn.mutateHeaderFunc = func(h http.Header) { + h.Set("Proxy-Authorization", pa) + } + } + case cm.targetScheme == "https": + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: cm.targetAddr}, + Host: cm.targetAddr, + Header: make(http.Header), + } + if pa != "" { + connectReq.Header.Set("Proxy-Authorization", pa) + } + connectReq.Write(conn) + + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, connectReq) + if err != nil { + conn.Close() + return nil, err + } + if resp.StatusCode != 200 { + f := strings.SplitN(resp.Status, " ", 2) + conn.Close() + return nil, errors.New(f[1]) + } + } + + if cm.targetScheme == "https" { + // Initiate TLS and check remote host name against certificate. + conn = tls.Client(conn, t.TLSClientConfig) + if err = conn.(*tls.Conn).Handshake(); err != nil { + return nil, err + } + if t.TLSClientConfig == nil || !t.TLSClientConfig.InsecureSkipVerify { + if err = conn.(*tls.Conn).VerifyHostname(cm.tlsHost()); err != nil { + return nil, err + } + } + pconn.conn = conn + } + + pconn.br = bufio.NewReader(pconn.conn) + pconn.bw = bufio.NewWriter(pconn.conn) + go pconn.readLoop() + return pconn, nil +} + +// useProxy returns true if requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() { + return false + } + } + + no_proxy := getenvEitherCase("NO_PROXY") + if no_proxy == "*" { + return false + } + + addr = strings.ToLower(strings.TrimSpace(addr)) + if hasPort(addr) { + addr = addr[:strings.LastIndex(addr, ":")] + } + + for _, p := range strings.Split(no_proxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + if hasPort(p) { + p = p[:strings.LastIndex(p, ":")] + } + if addr == p || (p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:])) { + return false + } + } + return true +} + +// connectMethod is the map key (in its String form) for keeping persistent +// TCP connections alive for subsequent HTTP requests. +// +// A connect method may be of the following types: +// +// Cache key form Description +// ----------------- ------------------------- +// ||http|foo.com http directly to server, no proxy +// ||https|foo.com https directly to server, no proxy +// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com +// http://proxy.com|http http to proxy, http to anywhere after that +// +// Note: no support to https to the proxy yet. +// +type connectMethod struct { + proxyURL *url.URL // nil for no proxy, else full proxy URL + targetScheme string // "http" or "https" + targetAddr string // Not used if proxy + http targetScheme (4th example in table) +} + +func (ck *connectMethod) String() string { + proxyStr := "" + if ck.proxyURL != nil { + proxyStr = ck.proxyURL.String() + } + return strings.Join([]string{proxyStr, ck.targetScheme, ck.targetAddr}, "|") +} + +// addr returns the first hop "host:port" to which we need to TCP connect. +func (cm *connectMethod) addr() string { + if cm.proxyURL != nil { + return canonicalAddr(cm.proxyURL) + } + return cm.targetAddr +} + +// tlsHost returns the host name to match against the peer's +// TLS certificate. +func (cm *connectMethod) tlsHost() string { + h := cm.targetAddr + if hasPort(h) { + h = h[:strings.LastIndex(h, ":")] + } + return h +} + +// persistConn wraps a connection, usually a persistent one +// (but may be used for non-keep-alive requests as well) +type persistConn struct { + t *Transport + cacheKey string // its connectMethod.String() + conn net.Conn + br *bufio.Reader // from conn + bw *bufio.Writer // to conn + reqch chan requestAndChan // written by roundTrip(); read by readLoop() + isProxy bool + + // mutateHeaderFunc is an optional func to modify extra + // headers on each outbound request before it's written. (the + // original Request given to RoundTrip is not modified) + mutateHeaderFunc func(http.Header) + + lk sync.Mutex // guards numExpectedResponses and broken + numExpectedResponses int + broken bool // an error has happened on this connection; marked broken so it's not reused. + + host string + ip *net.TCPAddr +} + +func (pc *persistConn) isBroken() bool { + pc.lk.Lock() + defer pc.lk.Unlock() + return pc.broken +} + +var remoteSideClosedFunc func(error) bool // or nil to use default + +func remoteSideClosed(err error) bool { + if err == io.EOF { + return true + } + if remoteSideClosedFunc != nil { + return remoteSideClosedFunc(err) + } + return false +} + +func (pc *persistConn) readLoop() { + alive := true + var lastbody io.ReadCloser // last response body, if any, read on this connection + + for alive { + pb, err := pc.br.Peek(1) + + pc.lk.Lock() + if pc.numExpectedResponses == 0 { + pc.closeLocked() + pc.lk.Unlock() + if len(pb) > 0 { + log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", + string(pb), err) + } + return + } + pc.lk.Unlock() + + rc := <-pc.reqch + + // Advance past the previous response's body, if the + // caller hasn't done so. + if lastbody != nil { + lastbody.Close() // assumed idempotent + lastbody = nil + } + resp, err := http.ReadResponse(pc.br, rc.req) + + if err != nil { + pc.close() + } else { + hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 + if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" { + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + gzReader, zerr := gzip.NewReader(resp.Body) + if zerr != nil { + pc.close() + err = zerr + } else { + resp.Body = &readFirstCloseBoth{&discardOnCloseReadCloser{gzReader}, resp.Body} + } + } + resp.Body = &bodyEOFSignal{body: resp.Body} + } + + if err != nil || resp.Close || rc.req.Close { + alive = false + } + + hasBody := resp != nil && resp.ContentLength != 0 + var waitForBodyRead chan bool + if alive { + if hasBody { + lastbody = resp.Body + waitForBodyRead = make(chan bool) + resp.Body.(*bodyEOFSignal).fn = func() { + if !pc.t.putIdleConn(pc) { + alive = false + } + waitForBodyRead <- true + } + } else { + // When there's no response body, we immediately + // reuse the TCP connection (putIdleConn), but + // we need to prevent ClientConn.Read from + // closing the Response.Body on the next + // loop, otherwise it might close the body + // before the client code has had a chance to + // read it (even though it'll just be 0, EOF). + lastbody = nil + + if !pc.t.putIdleConn(pc) { + alive = false + } + } + } + + rc.ch <- responseAndError{resp, err} + + // Wait for the just-returned response body to be fully consumed + // before we race and peek on the underlying bufio reader. + if waitForBodyRead != nil { + <-waitForBodyRead + } + } +} + +type responseAndError struct { + res *http.Response + err error +} + +type requestAndChan struct { + req *http.Request + ch chan responseAndError + + // did the Transport (as opposed to the client code) add an + // Accept-Encoding gzip header? only if it we set it do + // we transparently decode the gzip. + addedGzip bool +} + +func (pc *persistConn) roundTrip(req *transportRequest) (resp *http.Response, err error) { + if pc.mutateHeaderFunc != nil { + panic("mutateHeaderFunc not supported in modified Transport") + pc.mutateHeaderFunc(req.extraHeaders()) + } + + // Ask for a compressed version if the caller didn't set their + // own value for Accept-Encoding. We only attempted to + // uncompress the gzip stream if we were the layer that + // requested it. + requestedGzip := false + if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" { + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 + requestedGzip = true + req.extraHeaders().Set("Accept-Encoding", "gzip") + } + + pc.lk.Lock() + pc.numExpectedResponses++ + pc.lk.Unlock() + + // orig: err = req.Request.write(pc.bw, pc.isProxy, req.extra) + if pc.isProxy { + err = req.Request.WriteProxy(pc.bw) + } else { + err = req.Request.Write(pc.bw) + } + if err != nil { + pc.close() + return + } + pc.bw.Flush() + + ch := make(chan responseAndError, 1) + pc.reqch <- requestAndChan{req.Request, ch, requestedGzip} + re := <-ch + pc.lk.Lock() + pc.numExpectedResponses-- + pc.lk.Unlock() + + return re.res, re.err +} + +func (pc *persistConn) close() { + pc.lk.Lock() + defer pc.lk.Unlock() + pc.closeLocked() +} + +func (pc *persistConn) closeLocked() { + pc.broken = true + pc.conn.Close() + pc.mutateHeaderFunc = nil +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Host + if !hasPort(addr) { + return addr + ":" + portMap[url.Scheme] + } + return addr +} + +func responseIsKeepAlive(res *http.Response) bool { + // TODO: implement. for now just always shutting down the connection. + return false +} + +// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most +// once, right before the final Read() or Close() call returns, but after +// EOF has been seen. +type bodyEOFSignal struct { + body io.ReadCloser + fn func() + isClosed bool +} + +func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { + n, err = es.body.Read(p) + if es.isClosed && n > 0 { + panic("http: unexpected bodyEOFSignal Read after Close; see issue 1725") + } + if err == io.EOF && es.fn != nil { + es.fn() + es.fn = nil + } + return +} + +func (es *bodyEOFSignal) Close() (err error) { + if es.isClosed { + return nil + } + es.isClosed = true + err = es.body.Close() + if err == nil && es.fn != nil { + es.fn() + es.fn = nil + } + return +} + +type readFirstCloseBoth struct { + io.ReadCloser + io.Closer +} + +func (r *readFirstCloseBoth) Close() error { + if err := r.ReadCloser.Close(); err != nil { + r.Closer.Close() + return err + } + if err := r.Closer.Close(); err != nil { + return err + } + return nil +} + +// discardOnCloseReadCloser consumes all its input on Close. +type discardOnCloseReadCloser struct { + io.ReadCloser +} + +func (d *discardOnCloseReadCloser) Close() error { + io.Copy(ioutil.Discard, d.ReadCloser) // ignore errors; likely invalid or already closed + return d.ReadCloser.Close() +} diff --git a/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/util.go b/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/util.go new file mode 100644 index 000000000000..af0eda1ee685 --- /dev/null +++ b/Godeps/_workspace/src/github.com/elazarl/goproxy/transport/util.go @@ -0,0 +1,15 @@ +package transport + +import ( + "fmt" + "strings" +) + +type badStringError struct { + what string + str string +} + +func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } + +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } diff --git a/test/images/goproxy/.gitignore b/test/images/goproxy/.gitignore new file mode 100644 index 000000000000..663e1754bdef --- /dev/null +++ b/test/images/goproxy/.gitignore @@ -0,0 +1 @@ +goproxy diff --git a/test/images/goproxy/Dockerfile b/test/images/goproxy/Dockerfile new file mode 100644 index 000000000000..9aef17d92c8e --- /dev/null +++ b/test/images/goproxy/Dockerfile @@ -0,0 +1,17 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM scratch +ADD goproxy goproxy +ENTRYPOINT ["/goproxy"] diff --git a/test/images/goproxy/Makefile b/test/images/goproxy/Makefile new file mode 100644 index 000000000000..2f496ee7984a --- /dev/null +++ b/test/images/goproxy/Makefile @@ -0,0 +1,15 @@ +all: push + +TAG = 0.1 + +goproxy: goproxy.go + CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' ./goproxy.go + +image: goproxy + docker build -t gcr.io/google_containers/goproxy:$(TAG) . + +push: image + gcloud docker push gcr.io/google_containers/goproxy:$(TAG) + +clean: + rm -f goproxy diff --git a/test/images/goproxy/goproxy.go b/test/images/goproxy/goproxy.go new file mode 100644 index 000000000000..b3b1f6e24326 --- /dev/null +++ b/test/images/goproxy/goproxy.go @@ -0,0 +1,30 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "log" + "net/http" + + "github.com/elazarl/goproxy" +) + +func main() { + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + log.Fatal(http.ListenAndServe(":8080", proxy)) +} From 72b4a23c15e422f58dd8a1f759cc8269f35b294e Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Fri, 4 Sep 2015 10:15:35 -0400 Subject: [PATCH 03/13] url.URL.Host now canonical before use in spdy roundtripper. --- pkg/util/httpstream/spdy/roundtripper.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/util/httpstream/spdy/roundtripper.go b/pkg/util/httpstream/spdy/roundtripper.go index 35147edb2f15..b3307936fa46 100644 --- a/pkg/util/httpstream/spdy/roundtripper.go +++ b/pkg/util/httpstream/spdy/roundtripper.go @@ -79,11 +79,14 @@ func (s *SpdyRoundTripper) dial(req *http.Request) (net.Conn, error) { return s.dialWithoutProxy(req.URL) } + // ensure we use a canonical host with proxyReq + targetHost := netutil.CanonicalAddr(req.URL) + // proxying logic adapted from http://blog.h6t.eu/post/74098062923/golang-websocket-with-http-proxy-support proxyReq := http.Request{ Method: "CONNECT", URL: &url.URL{}, - Host: req.URL.Host, + Host: targetHost, } proxyDialConn, err := s.dialWithoutProxy(proxyURL) From 00deaf444f525a4d1bd8bf8a07adfaf9b85e8e55 Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Thu, 17 Sep 2015 11:19:48 -0400 Subject: [PATCH 04/13] netexec: Multiple fixes and enhancements to netexec * Added upload functionality * More logging * Moved to test/images * Image file fixes --- .../images}/netexec/.gitignore | 0 .../images}/netexec/Dockerfile | 4 ++ .../images}/netexec/Makefile | 0 .../images}/netexec/netexec.go | 40 +++++++++++++++++++ test/images/netexec/pod.yaml | 13 ++++++ 5 files changed, 57 insertions(+) rename {contrib/for-tests => test/images}/netexec/.gitignore (100%) rename {contrib/for-tests => test/images}/netexec/Dockerfile (74%) rename {contrib/for-tests => test/images}/netexec/Makefile (100%) rename {contrib/for-tests => test/images}/netexec/netexec.go (84%) create mode 100644 test/images/netexec/pod.yaml diff --git a/contrib/for-tests/netexec/.gitignore b/test/images/netexec/.gitignore similarity index 100% rename from contrib/for-tests/netexec/.gitignore rename to test/images/netexec/.gitignore diff --git a/contrib/for-tests/netexec/Dockerfile b/test/images/netexec/Dockerfile similarity index 74% rename from contrib/for-tests/netexec/Dockerfile rename to test/images/netexec/Dockerfile index b2eaa4d0c190..9c57cac41504 100644 --- a/contrib/for-tests/netexec/Dockerfile +++ b/test/images/netexec/Dockerfile @@ -3,5 +3,9 @@ MAINTAINER Abhishek Shah "abshah@google.com" ADD netexec netexec ADD netexec.go netexec.go +EXPOSE 8080 +EXPOSE 8081 + +RUN mkdir /uploads ENTRYPOINT ["/netexec"] diff --git a/contrib/for-tests/netexec/Makefile b/test/images/netexec/Makefile similarity index 100% rename from contrib/for-tests/netexec/Makefile rename to test/images/netexec/Makefile diff --git a/contrib/for-tests/netexec/netexec.go b/test/images/netexec/netexec.go similarity index 84% rename from contrib/for-tests/netexec/netexec.go rename to test/images/netexec/netexec.go index cfca4a129715..7ab30be33ee7 100644 --- a/contrib/for-tests/netexec/netexec.go +++ b/test/images/netexec/netexec.go @@ -20,6 +20,7 @@ import ( "encoding/json" "flag" "fmt" + "io" "io/ioutil" "log" "net" @@ -58,6 +59,7 @@ func startHTTPServer(httpPort int) { http.HandleFunc("/shutdown", shutdownHandler) http.HandleFunc("/hostName", hostNameHandler) http.HandleFunc("/shell", shellHandler) + http.HandleFunc("/upload", uploadHandler) http.HandleFunc("/dial", dialHandler) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", httpPort), nil)) } @@ -191,6 +193,7 @@ func dialUDP(request string, remoteAddress *net.UDPAddr) (string, error) { func shellHandler(w http.ResponseWriter, r *http.Request) { log.Println(r.FormValue("shellCommand")) + log.Printf("%s %s %s\n", shellPath, "-c", r.FormValue("shellCommand")) cmdOut, err := exec.Command(shellPath, "-c", r.FormValue("shellCommand")).CombinedOutput() output := map[string]string{} if len(cmdOut) > 0 { @@ -207,6 +210,43 @@ func shellHandler(w http.ResponseWriter, r *http.Request) { } } +func uploadHandler(w http.ResponseWriter, r *http.Request) { + file, _, err := r.FormFile("file") + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Unable to upload file.") + log.Printf("Unable to upload file: %s", err) + return + } + defer file.Close() + + f, err := ioutil.TempFile("/uploads", "upload") + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Unable to open file for write.") + log.Printf("Unable to open file for write: %s", err) + return + } + defer f.Close() + if _, err = io.Copy(f, file); err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Unable to write file.")) + log.Printf("Unable to write file: %s", err) + return + } + + UploadFile := f.Name() + if err := os.Chmod(UploadFile, 0700); err != nil { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Unable to chmod file.") + log.Printf("Unable to chmod file: %s", err) + return + } + log.Printf("Wrote upload to %s", UploadFile) + w.WriteHeader(http.StatusCreated) + fmt.Fprintf(w, UploadFile) +} + func hostNameHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, getHostName()) } diff --git a/test/images/netexec/pod.yaml b/test/images/netexec/pod.yaml new file mode 100644 index 000000000000..1f8782402acb --- /dev/null +++ b/test/images/netexec/pod.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: netexec + labels: + app: netexec +spec: + containers: + - name: netexec + image: gcr.io/google_containers/netexec:1.1 + ports: + - containerPort: 8080 + - containerPort: 8081 From ba5735559ee6611a4d68847af10bf5d4c3761e7c Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Thu, 17 Sep 2015 16:11:27 -0400 Subject: [PATCH 05/13] e2e: kubectl verification for HTTP proxying using netexec and goproxy. --- test/e2e/kubectl.go | 207 ++++++++++++++++++++++++++++++++- test/images/goproxy/Dockerfile | 1 + test/images/goproxy/pod.yaml | 12 ++ test/images/netexec/Makefile | 3 +- test/images/netexec/netexec.go | 42 +++++-- test/images/netexec/pod.yaml | 2 +- 6 files changed, 253 insertions(+), 14 deletions(-) create mode 100644 test/images/goproxy/pod.yaml diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index 07202955265e..4e366033e9be 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "io/ioutil" + "mime/multipart" "net" "net/http" "os" @@ -56,6 +57,10 @@ const ( frontendSelector = "name=frontend" redisMasterSelector = "name=redis-master" redisSlaveSelector = "name=redis-slave" + goproxyContainer = "goproxy" + goproxyPodSelector = "name=goproxy" + netexecContainer = "netexec" + netexecPodSelector = "name=netexec" kubectlProxyPort = 8011 guestbookStartupTimeout = 10 * time.Minute guestbookResponseTimeout = 3 * time.Minute @@ -154,7 +159,6 @@ var _ = Describe("Kubectl client", func() { By("creating the pod") runKubectl("create", "-f", podPath, fmt.Sprintf("--namespace=%v", ns)) checkPodsRunningReady(c, ns, []string{simplePodName}, podStartTimeout) - }) AfterEach(func() { cleanup(podPath, ns, simplePodSelector) @@ -176,12 +180,12 @@ var _ = Describe("Kubectl client", func() { } // pretend that we're a user in an interactive shell - r, c, err := newBlockingReader("echo hi\nexit\n") + r, closer, err := newBlockingReader("echo hi\nexit\n") if err != nil { Failf("Error creating blocking reader: %v", err) } // NOTE this is solely for test cleanup! - defer c.Close() + defer closer.Close() By("executing a command in the container with pseudo-interactive stdin") execOutput = newKubectlCommand("exec", fmt.Sprintf("--namespace=%v", ns), "-i", simplePodName, "bash"). @@ -192,6 +196,163 @@ var _ = Describe("Kubectl client", func() { } }) + It("should support exec through an HTTP proxy", func() { + // Note: We are skipping local since we want to verify an apiserver with HTTPS. + // At this time local only supports plain HTTP. + SkipIfProviderIs("local") + // Fail if the variable isn't set + if testContext.Host == "" { + Failf("--host variable must be set to the full URI to the api server on e2e run.") + } + apiServer := testContext.Host + // If there is no api in URL try to add it + if !strings.Contains(apiServer, ":443/api") { + apiServer = apiServer + ":443/api" + } + + // Get the kube/config + testWorkspace := os.Getenv("WORKSPACE") + if testWorkspace == "" { + // Not running in jenkins, assume "HOME" + testWorkspace = os.Getenv("HOME") + } + + testKubectlPath := testContext.KubectlPath + // If no path is given then default to Jenkins e2e expected path + if testKubectlPath == "" || testKubectlPath == "kubectl" { + testKubectlPath = filepath.Join(testWorkspace, "kubernetes", "platforms", "linux", "amd64", "kubectl") + } + // Get the kubeconfig path + kubeConfigFilePath := testContext.KubeConfig + if kubeConfigFilePath == "" { + // Fall back to the jenkins e2e location + kubeConfigFilePath = filepath.Join(testWorkspace, ".kube", "config") + } + + _, err := os.Stat(kubeConfigFilePath) + if err != nil { + Failf("kube config path could not be accessed. Error=%s", err) + } + // start exec-proxy-tester container + netexecPodPath := filepath.Join(testContext.RepoRoot, "test/images/netexec/pod.yaml") + runKubectl("create", "-f", netexecPodPath, fmt.Sprintf("--namespace=%v", ns)) + checkPodsRunningReady(c, ns, []string{netexecContainer}, podStartTimeout) + // Clean up + defer cleanup(netexecPodPath, ns, netexecPodSelector) + // Upload kubeconfig + type NetexecOutput struct { + Output string `json:"output"` + Error string `json:"error"` + } + + var uploadConfigOutput NetexecOutput + // Upload the kubeconfig file + By("uploading kubeconfig to netexec") + pipeConfigReader, postConfigBodyWriter, err := newStreamingUpload(kubeConfigFilePath) + if err != nil { + Failf("unable to create streaming upload. Error: %s", err) + } + + resp, err := c.Post(). + Prefix("proxy"). + Namespace(ns). + Name("netexec"). + Resource("pods"). + Suffix("upload"). + SetHeader("Content-Type", postConfigBodyWriter.FormDataContentType()). + Body(pipeConfigReader). + Do().Raw() + if err != nil { + Failf("Unable to upload kubeconfig to the remote exec server due to error: %s", err) + } + + if err := json.Unmarshal(resp, &uploadConfigOutput); err != nil { + Failf("Unable to read the result from the netexec server. Error: %s", err) + } + kubecConfigRemotePath := uploadConfigOutput.Output + + // Upload + pipeReader, postBodyWriter, err := newStreamingUpload(testContext.KubectlPath) + if err != nil { + Failf("unable to create streaming upload. Error: %s", err) + } + + By("uploading kubectl to netexec") + var uploadOutput NetexecOutput + // Upload the kubectl binary + resp, err = c.Post(). + Prefix("proxy"). + Namespace(ns). + Name("netexec"). + Resource("pods"). + Suffix("upload"). + SetHeader("Content-Type", postBodyWriter.FormDataContentType()). + Body(pipeReader). + Do().Raw() + if err != nil { + Failf("Unable to upload kubectl binary to the remote exec server due to error: %s", err) + } + + if err := json.Unmarshal(resp, &uploadOutput); err != nil { + Failf("Unable to read the result from the netexec server. Error: %s", err) + } + uploadBinaryName := uploadOutput.Output + // Verify that we got the expected response back in the body + if !strings.HasPrefix(uploadBinaryName, "/uploads/") { + Failf("Unable to upload kubectl binary to remote exec server. /uploads/ not in response. Response: %s", uploadBinaryName) + } + + for _, proxyVar := range []string{"https_proxy", "HTTPS_PROXY"} { + By("Running kubectl in netexec via an HTTP proxy using " + proxyVar) + // start the proxy container + goproxyPodPath := filepath.Join(testContext.RepoRoot, "test/images/goproxy/pod.yaml") + runKubectl("create", "-f", goproxyPodPath, fmt.Sprintf("--namespace=%v", ns)) + checkPodsRunningReady(c, ns, []string{goproxyContainer}, podStartTimeout) + + // get the proxy address + goproxyPod, err := c.Pods(ns).Get(goproxyContainer) + if err != nil { + Failf("Unable to get the goproxy pod. Error: %s", err) + } + proxyAddr := fmt.Sprintf("http://%s:8080", goproxyPod.Status.PodIP) + + shellCommand := fmt.Sprintf("%s=%s .%s --kubeconfig=%s --server=%s --namespace=%s exec nginx echo running in container", proxyVar, proxyAddr, uploadBinaryName, kubecConfigRemotePath, apiServer, ns) + // Execute kubectl on remote exec server. + netexecShellOutput, err := c.Post(). + Prefix("proxy"). + Namespace(ns). + Name("netexec"). + Resource("pods"). + Suffix("shell"). + Param("shellCommand", shellCommand). + Do().Raw() + if err != nil { + Failf("Unable to execute kubectl binary on the remote exec server due to error: %s", err) + } + + var netexecOuput NetexecOutput + if err := json.Unmarshal(netexecShellOutput, &netexecOuput); err != nil { + Failf("Unable to read the result from the netexec server. Error: %s", err) + } + + // Verify we got the normal output captured by the exec server + expectedExecOutput := "running in container\n" + if netexecOuput.Output != expectedExecOutput { + Failf("Unexpected kubectl exec output. Wanted %q, got %q", expectedExecOutput, netexecOuput.Output) + } + + // Verify the proxy server logs saw the connection + expectedProxyLog := fmt.Sprintf("Accepting CONNECT to %s", strings.TrimRight(strings.TrimLeft(testContext.Host, "https://"), "/api")) + proxyLog := runKubectl("log", "goproxy", fmt.Sprintf("--namespace=%v", ns)) + + if !strings.Contains(proxyLog, expectedProxyLog) { + Failf("Missing expected log result on proxy server for %s. Expected: %q, got %q", proxyVar, expectedProxyLog, proxyLog) + } + // Clean up the goproxyPod + cleanup(goproxyPodPath, ns, goproxyPodSelector) + } + }) + It("should support inline execution and attach", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) @@ -993,3 +1154,43 @@ func newBlockingReader(s string) (io.Reader, io.Closer, error) { w.Write([]byte(s)) return r, w, nil } + +// newStreamingUpload creates a new http.Request that will stream POST +// a file to a URI. +func newStreamingUpload(filePath string) (*io.PipeReader, *multipart.Writer, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, nil, err + } + + r, w := io.Pipe() + + postBodyWriter := multipart.NewWriter(w) + + go streamingUpload(file, filepath.Base(filePath), postBodyWriter, w) + return r, postBodyWriter, err +} + +// streamingUpload streams a file via a pipe through a multipart.Writer. +// Generally one should use newStreamingUpload instead of calling this directly. +func streamingUpload(file *os.File, fileName string, postBodyWriter *multipart.Writer, w *io.PipeWriter) { + defer GinkgoRecover() + defer file.Close() + defer w.Close() + + // Set up the form file + fileWriter, err := postBodyWriter.CreateFormFile("file", fileName) + if err != nil { + Failf("Unable to to write file at %s to buffer. Error: %s", fileName, err) + } + + // Copy kubectl binary into the file writer + if _, err := io.Copy(fileWriter, file); err != nil { + Failf("Unable to to copy file at %s into the file writer. Error: %s", fileName, err) + } + + // Nothing more should be written to this instance of the postBodyWriter + if err := postBodyWriter.Close(); err != nil { + Failf("Unable to close the writer for file upload. Error: %s", err) + } +} diff --git a/test/images/goproxy/Dockerfile b/test/images/goproxy/Dockerfile index 9aef17d92c8e..aaa18fa1185a 100644 --- a/test/images/goproxy/Dockerfile +++ b/test/images/goproxy/Dockerfile @@ -14,4 +14,5 @@ FROM scratch ADD goproxy goproxy +EXPOSE 8080 ENTRYPOINT ["/goproxy"] diff --git a/test/images/goproxy/pod.yaml b/test/images/goproxy/pod.yaml new file mode 100644 index 000000000000..47baf2e2e703 --- /dev/null +++ b/test/images/goproxy/pod.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: goproxy + labels: + app: goproxy +spec: + containers: + - name: goproxy + image: gcr.io/google_containers/goproxy:0.1 + ports: + - containerPort: 8080 diff --git a/test/images/netexec/Makefile b/test/images/netexec/Makefile index 57098585af30..605375dc63a3 100644 --- a/test/images/netexec/Makefile +++ b/test/images/netexec/Makefile @@ -1,8 +1,9 @@ .PHONY: all netexec image push clean -TAG = 1.1 +TAG = 1.3.1 PREFIX = gcr.io/google_containers + all: push netexec: netexec.go diff --git a/test/images/netexec/netexec.go b/test/images/netexec/netexec.go index 7ab30be33ee7..62e8b59f6cb7 100644 --- a/test/images/netexec/netexec.go +++ b/test/images/netexec/netexec.go @@ -202,6 +202,7 @@ func shellHandler(w http.ResponseWriter, r *http.Request) { if err != nil { output["error"] = fmt.Sprintf("%v", err) } + log.Printf("Output: %s", output) bytes, err := json.Marshal(output) if err == nil { fmt.Fprintf(w, string(bytes)) @@ -211,10 +212,16 @@ func shellHandler(w http.ResponseWriter, r *http.Request) { } func uploadHandler(w http.ResponseWriter, r *http.Request) { + result := map[string]string{} file, _, err := r.FormFile("file") if err != nil { - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "Unable to upload file.") + result["error"] = "Unable to upload file." + bytes, err := json.Marshal(result) + if err == nil { + fmt.Fprintf(w, string(bytes)) + } else { + http.Error(w, fmt.Sprintf("%s. Also unable to serialize output. %v", result["error"], err), http.StatusInternalServerError) + } log.Printf("Unable to upload file: %s", err) return } @@ -222,29 +229,46 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { f, err := ioutil.TempFile("/uploads", "upload") if err != nil { - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "Unable to open file for write.") + result["error"] = "Unable to open file for write" + bytes, err := json.Marshal(result) + if err == nil { + fmt.Fprintf(w, string(bytes)) + } else { + http.Error(w, fmt.Sprintf("%s. Also unable to serialize output. %v", result["error"], err), http.StatusInternalServerError) + } log.Printf("Unable to open file for write: %s", err) return } defer f.Close() if _, err = io.Copy(f, file); err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("Unable to write file.")) + result["error"] = "Unable to write file." + bytes, err := json.Marshal(result) + if err == nil { + fmt.Fprintf(w, string(bytes)) + } else { + http.Error(w, fmt.Sprintf("%s. Also unable to serialize output. %v", result["error"], err), http.StatusInternalServerError) + } log.Printf("Unable to write file: %s", err) return } UploadFile := f.Name() if err := os.Chmod(UploadFile, 0700); err != nil { - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "Unable to chmod file.") + result["error"] = "Unable to chmod file." + bytes, err := json.Marshal(result) + if err == nil { + fmt.Fprintf(w, string(bytes)) + } else { + http.Error(w, fmt.Sprintf("%s. Also unable to serialize output. %v", result["error"], err), http.StatusInternalServerError) + } log.Printf("Unable to chmod file: %s", err) return } log.Printf("Wrote upload to %s", UploadFile) + result["output"] = UploadFile w.WriteHeader(http.StatusCreated) - fmt.Fprintf(w, UploadFile) + bytes, err := json.Marshal(result) + fmt.Fprintf(w, string(bytes)) } func hostNameHandler(w http.ResponseWriter, r *http.Request) { diff --git a/test/images/netexec/pod.yaml b/test/images/netexec/pod.yaml index 1f8782402acb..eb2273d40f20 100644 --- a/test/images/netexec/pod.yaml +++ b/test/images/netexec/pod.yaml @@ -7,7 +7,7 @@ metadata: spec: containers: - name: netexec - image: gcr.io/google_containers/netexec:1.1 + image: gcr.io/google_containers/netexec:1.3.1 ports: - containerPort: 8080 - containerPort: 8081 From 5af63add51699f1779b8a2fae43a31c4f9f3e890 Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Tue, 29 Sep 2015 11:20:49 -0400 Subject: [PATCH 06/13] build: test/images in test tar and a static kubectl * release tar now includes test/images/* * kubectl is now built as a static binary in the test --- build/common.sh | 3 +++ test/e2e/kubectl.go | 33 +++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/build/common.sh b/build/common.sh index fafa59ce73e7..7e0c203320b4 100755 --- a/build/common.sh +++ b/build/common.sh @@ -832,6 +832,9 @@ function kube::release::package_test_tarball() { "${release_stage}/platforms/${platform}" done + # Add the test image files + mkdir -p "${release_stage}/test/images" + cp -fR "${KUBE_ROOT}/test/images" "${release_stage}/test/" tar c ${KUBE_TEST_PORTABLE[@]} | tar x -C ${release_stage} kube::release::clean_cruft diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index 4e366033e9be..d568e3335530 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -28,6 +28,7 @@ import ( "net/http" "os" "os/exec" + "path" "path/filepath" "regexp" "strconv" @@ -211,25 +212,30 @@ var _ = Describe("Kubectl client", func() { } // Get the kube/config + // TODO: Can it be RepoRoot with jenkins e2e? testWorkspace := os.Getenv("WORKSPACE") if testWorkspace == "" { - // Not running in jenkins, assume "HOME" - testWorkspace = os.Getenv("HOME") + // Not running in jenkins, assume RepoRoot + testWorkspace = testContext.RepoRoot // os.Getenv("HOME") } - testKubectlPath := testContext.KubectlPath - // If no path is given then default to Jenkins e2e expected path - if testKubectlPath == "" || testKubectlPath == "kubectl" { - testKubectlPath = filepath.Join(testWorkspace, "kubernetes", "platforms", "linux", "amd64", "kubectl") + // Build the static kubectl + By("Building a static kubectl for upload") + kubectlContainerPath := path.Join(testWorkspace, "/examples/kubectl-container/") + staticKubectlBuild := exec.Command("make", "-C", kubectlContainerPath) + if out, err := staticKubectlBuild.Output(); err != nil { + Failf("Unable to create static kubectl. Error=%s, Output=%s", err, out) } - // Get the kubeconfig path - kubeConfigFilePath := testContext.KubeConfig - if kubeConfigFilePath == "" { - // Fall back to the jenkins e2e location - kubeConfigFilePath = filepath.Join(testWorkspace, ".kube", "config") + // Verify the static kubectl path + testStaticKubectlPath := path.Join(kubectlContainerPath, "kubectl") + _, err := os.Stat(testStaticKubectlPath) + if err != nil { + Failf("static kubectl path could not be accessed. Error=%s", err) } - _, err := os.Stat(kubeConfigFilePath) + // Verify the kubeconfig path + kubeConfigFilePath := testContext.KubeConfig + _, err = os.Stat(kubeConfigFilePath) if err != nil { Failf("kube config path could not be accessed. Error=%s", err) } @@ -252,7 +258,6 @@ var _ = Describe("Kubectl client", func() { if err != nil { Failf("unable to create streaming upload. Error: %s", err) } - resp, err := c.Post(). Prefix("proxy"). Namespace(ns). @@ -272,7 +277,7 @@ var _ = Describe("Kubectl client", func() { kubecConfigRemotePath := uploadConfigOutput.Output // Upload - pipeReader, postBodyWriter, err := newStreamingUpload(testContext.KubectlPath) + pipeReader, postBodyWriter, err := newStreamingUpload(testStaticKubectlPath) if err != nil { Failf("unable to create streaming upload. Error: %s", err) } From efe2de91f5759e3492b8afffcbed141e10b75416 Mon Sep 17 00:00:00 2001 From: Marcin Wielgus Date: Fri, 2 Oct 2015 17:22:31 +0200 Subject: [PATCH 07/13] Wait for at least 1 endpoint in E2E test for examples --- test/e2e/examples.go | 19 +++++++++++++++---- test/e2e/util.go | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/test/e2e/examples.go b/test/e2e/examples.go index fcd27fd2e4e5..23ed0be285dd 100644 --- a/test/e2e/examples.go +++ b/test/e2e/examples.go @@ -34,10 +34,9 @@ import ( ) const ( - podListTimeout = time.Minute - serverStartTimeout = podStartTimeout + 3*time.Minute - dnsReadyTimeout = time.Minute - endpointRegisterTimeout = time.Minute + podListTimeout = time.Minute + serverStartTimeout = podStartTimeout + 3*time.Minute + dnsReadyTimeout = time.Minute ) const queryDnsPythonTemplate string = ` @@ -149,6 +148,8 @@ var _ = Describe("Examples e2e", func() { _, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) }) + waitForEndpoint(c, ns, "rabbitmq-service") + By("starting celery") runKubectl("create", "-f", celeryControllerYaml, nsFlag) forEachPod(c, ns, "component", "celery", func(pod api.Pod) { @@ -192,6 +193,9 @@ var _ = Describe("Examples e2e", func() { _, err = lookForStringInLog(ns, "spark-driver", "spark-driver", "Use kubectl exec", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) + By("waiting for master endpoint") + waitForEndpoint(c, ns, "spark-master") + By("starting workers") runKubectl("create", "-f", workerControllerJson, nsFlag) ScaleRC(c, ns, "spark-worker-controller", 2, true) @@ -221,6 +225,8 @@ var _ = Describe("Examples e2e", func() { _, err = lookForStringInLog(ns, "cassandra", "cassandra", "Listening for thrift clients", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) + waitForEndpoint(c, ns, "cassandra") + By("create and scale rc") runKubectl("create", "-f", controllerYaml, nsFlag) err = ScaleRC(c, ns, "cassandra", 2, true) @@ -263,12 +269,14 @@ var _ = Describe("Examples e2e", func() { By("checking if zookeeper is up and running") _, err = lookForStringInLog(ns, zookeeperPod, "zookeeper", "binding to port", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) + waitForEndpoint(c, ns, "zookeeper") By("starting Nimbus") runKubectl("create", "-f", nimbusPodJson, nsFlag) runKubectl("create", "-f", nimbusServiceJson, nsFlag) err = waitForPodRunningInNamespace(c, "nimbus", ns) Expect(err).NotTo(HaveOccurred()) + waitForEndpoint(c, ns, "nimbus") By("starting workers") runKubectl("create", "-f", workerControllerJson, nsFlag) @@ -382,6 +390,7 @@ var _ = Describe("Examples e2e", func() { }) } checkDbInstances() + waitForEndpoint(c, ns, "rethinkdb-driver") By("scaling rethinkdb") ScaleRC(c, ns, "rethinkdb-rc", 2, true) @@ -420,6 +429,8 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) }) + waitForEndpoint(c, ns, "hazelcast") + By("scaling hazelcast") ScaleRC(c, ns, "hazelcast", 2, true) forEachPod(c, ns, "name", "hazelcast", func(pod api.Pod) { diff --git a/test/e2e/util.go b/test/e2e/util.go index 571f61478c86..343a7fea4b24 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -94,6 +94,7 @@ const ( podRespondingTimeout = 2 * time.Minute serviceRespondingTimeout = 2 * time.Minute + endpointRegisterTimeout = time.Minute // How wide to print pod names, by default. Useful for aligning printing to // quickly scan through output. @@ -731,6 +732,20 @@ func waitForReplicationController(c *client.Client, namespace, name string, exis return nil } +func waitForEndpoint(c *client.Client, ns, name string) error { + for t := time.Now(); time.Since(t) < endpointRegisterTimeout; time.Sleep(poll) { + endpoint, err := c.Endpoints(ns).Get(name) + Expect(err).NotTo(HaveOccurred()) + if len(endpoint.Subsets) == 0 || len(endpoint.Subsets[0].Addresses) == 0 { + Logf("Endpoint %s/%s is not ready yet", ns, name) + continue + } else { + return nil + } + } + return fmt.Errorf("Failed to get entpoints for %s/%s", ns, name) +} + // Context for checking pods responses by issuing GETs to them and verifying if the answer with pod name. type podResponseChecker struct { c *client.Client From dae8d7aeb654f24df433be084a3937f105045680 Mon Sep 17 00:00:00 2001 From: Marcin Wielgus Date: Tue, 6 Oct 2015 13:16:20 +0200 Subject: [PATCH 08/13] Error check for waitForEndoint in e2e examples tests --- test/e2e/examples.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/test/e2e/examples.go b/test/e2e/examples.go index 23ed0be285dd..11f065f8d3dc 100644 --- a/test/e2e/examples.go +++ b/test/e2e/examples.go @@ -148,7 +148,8 @@ var _ = Describe("Examples e2e", func() { _, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) }) - waitForEndpoint(c, ns, "rabbitmq-service") + err := waitForEndpoint(c, ns, "rabbitmq-service") + Expect(err).NotTo(HaveOccurred()) By("starting celery") runKubectl("create", "-f", celeryControllerYaml, nsFlag) @@ -194,7 +195,8 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("waiting for master endpoint") - waitForEndpoint(c, ns, "spark-master") + err = waitForEndpoint(c, ns, "spark-master") + Expect(err).NotTo(HaveOccurred()) By("starting workers") runKubectl("create", "-f", workerControllerJson, nsFlag) @@ -225,7 +227,8 @@ var _ = Describe("Examples e2e", func() { _, err = lookForStringInLog(ns, "cassandra", "cassandra", "Listening for thrift clients", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) - waitForEndpoint(c, ns, "cassandra") + err = waitForEndpoint(c, ns, "cassandra") + Expect(err).NotTo(HaveOccurred()) By("create and scale rc") runKubectl("create", "-f", controllerYaml, nsFlag) @@ -269,14 +272,17 @@ var _ = Describe("Examples e2e", func() { By("checking if zookeeper is up and running") _, err = lookForStringInLog(ns, zookeeperPod, "zookeeper", "binding to port", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) - waitForEndpoint(c, ns, "zookeeper") + err = waitForEndpoint(c, ns, "zookeeper") + Expect(err).NotTo(HaveOccurred()) By("starting Nimbus") runKubectl("create", "-f", nimbusPodJson, nsFlag) runKubectl("create", "-f", nimbusServiceJson, nsFlag) err = waitForPodRunningInNamespace(c, "nimbus", ns) Expect(err).NotTo(HaveOccurred()) - waitForEndpoint(c, ns, "nimbus") + + err = waitForEndpoint(c, ns, "nimbus") + Expect(err).NotTo(HaveOccurred()) By("starting workers") runKubectl("create", "-f", workerControllerJson, nsFlag) @@ -390,7 +396,8 @@ var _ = Describe("Examples e2e", func() { }) } checkDbInstances() - waitForEndpoint(c, ns, "rethinkdb-driver") + err := waitForEndpoint(c, ns, "rethinkdb-driver") + Expect(err).NotTo(HaveOccurred()) By("scaling rethinkdb") ScaleRC(c, ns, "rethinkdb-rc", 2, true) @@ -399,7 +406,7 @@ var _ = Describe("Examples e2e", func() { By("starting admin") runKubectl("create", "-f", adminServiceYaml, nsFlag) runKubectl("create", "-f", adminPodYaml, nsFlag) - err := waitForPodRunningInNamespace(c, "rethinkdb-admin", ns) + err = waitForPodRunningInNamespace(c, "rethinkdb-admin", ns) Expect(err).NotTo(HaveOccurred()) checkDbInstances() content, err := makeHttpRequestToService(c, ns, "rethinkdb-admin", "/", endpointRegisterTimeout) @@ -429,7 +436,8 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) }) - waitForEndpoint(c, ns, "hazelcast") + err := waitForEndpoint(c, ns, "hazelcast") + Expect(err).NotTo(HaveOccurred()) By("scaling hazelcast") ScaleRC(c, ns, "hazelcast", 2, true) From c2e677ab18f078d36c116040a6387769d7bdcbed Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sun, 11 Oct 2015 12:58:56 +0200 Subject: [PATCH 09/13] Let port forwarding e2e tests timeout on kubectl calls --- test/e2e/portforward.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/e2e/portforward.go b/test/e2e/portforward.go index 2b5a2ca5baef..2acb59450bc2 100644 --- a/test/e2e/portforward.go +++ b/test/e2e/portforward.go @@ -24,8 +24,10 @@ import ( "regexp" "strconv" "strings" + "time" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/util" . "github.com/onsi/ginkgo" ) @@ -110,6 +112,20 @@ func runPortForward(ns, podName string, port int) (*exec.Cmd, int) { return cmd, listenPort } +func runKubectlWithTimeout(timeout time.Duration, args ...string) string { + logOutput := make(chan string) + go func() { + logOutput <- runKubectl(args...) + }() + select { + case <-time.After(timeout): + Failf("kubectl timed out") + return "" + case o := <-logOutput: + return o + } +} + var _ = Describe("Port forwarding", func() { framework := NewFramework("port-forwarding") @@ -133,7 +149,7 @@ var _ = Describe("Port forwarding", func() { By("Closing the connection to the local port") conn.Close() - logOutput := runKubectl("logs", fmt.Sprintf("--namespace=%v", framework.Namespace.Name), "-f", podName) + logOutput := runKubectlWithTimeout(util.ForeverTestTimeout, "logs", fmt.Sprintf("--namespace=%v", framework.Namespace.Name), "-f", podName) verifyLogMessage(logOutput, "Accepted client connection") verifyLogMessage(logOutput, "Expected to read 3 bytes from client, but got 0 instead") }) @@ -178,7 +194,7 @@ var _ = Describe("Port forwarding", func() { Failf("Expected %q from server, got %q", e, a) } - logOutput := runKubectl("logs", fmt.Sprintf("--namespace=%v", framework.Namespace.Name), "-f", podName) + logOutput := runKubectlWithTimeout(util.ForeverTestTimeout, "logs", fmt.Sprintf("--namespace=%v", framework.Namespace.Name), "-f", podName) verifyLogMessage(logOutput, "^Accepted client connection$") verifyLogMessage(logOutput, "^Received expected client data$") verifyLogMessage(logOutput, "^Done$") @@ -215,7 +231,7 @@ var _ = Describe("Port forwarding", func() { Failf("Expected %q from server, got %q", e, a) } - logOutput := runKubectl("logs", fmt.Sprintf("--namespace=%v", framework.Namespace.Name), "-f", podName) + logOutput := runKubectlWithTimeout(util.ForeverTestTimeout, "logs", fmt.Sprintf("--namespace=%v", framework.Namespace.Name), "-f", podName) verifyLogMessage(logOutput, "Accepted client connection") verifyLogMessage(logOutput, "Done") }) From a5561e6b173e5f736c9a5b215628f5f7d5ec0ba8 Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Mon, 19 Oct 2015 14:17:02 -0400 Subject: [PATCH 10/13] e2e: kubectl HTTP fix for #15713 and #15787. This reverts commit 54c899370ee942fdda646bf3e199cc9b97cb9fb0, reversing changes made to 0338e0ef5383037dcba5807ac8047c6b306e14c3 and adds a fix proposed in #15848. --- test/e2e/kubectl.go | 52 ++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index d568e3335530..2d86da7763c9 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -26,6 +26,7 @@ import ( "mime/multipart" "net" "net/http" + "net/url" "os" "os/exec" "path" @@ -205,33 +206,44 @@ var _ = Describe("Kubectl client", func() { if testContext.Host == "" { Failf("--host variable must be set to the full URI to the api server on e2e run.") } + + // Make sure the apiServer is set to what kubectl requires apiServer := testContext.Host - // If there is no api in URL try to add it - if !strings.Contains(apiServer, ":443/api") { - apiServer = apiServer + ":443/api" + apiServerUrl, err := url.Parse(apiServer) + if err != nil { + Failf("Unable to parse URL %s. Error=%s", apiServer, err) } - - // Get the kube/config - // TODO: Can it be RepoRoot with jenkins e2e? - testWorkspace := os.Getenv("WORKSPACE") - if testWorkspace == "" { - // Not running in jenkins, assume RepoRoot - testWorkspace = testContext.RepoRoot // os.Getenv("HOME") + apiServerUrl.Scheme = "https" + apiServerUrl.Path = "/api" + if !strings.Contains(apiServer, ":443") { + apiServerUrl.Host = apiServerUrl.Host + ":443" } + apiServer = apiServerUrl.String() // Build the static kubectl - By("Building a static kubectl for upload") - kubectlContainerPath := path.Join(testWorkspace, "/examples/kubectl-container/") - staticKubectlBuild := exec.Command("make", "-C", kubectlContainerPath) - if out, err := staticKubectlBuild.Output(); err != nil { - Failf("Unable to create static kubectl. Error=%s, Output=%s", err, out) - } - // Verify the static kubectl path - testStaticKubectlPath := path.Join(kubectlContainerPath, "kubectl") - _, err := os.Stat(testStaticKubectlPath) + By("Finding a static kubectl for upload") + testStaticKubectlPath := path.Join(testContext.RepoRoot, "platforms/linux/386/kubectl") + _, err = os.Stat(testStaticKubectlPath) if err != nil { - Failf("static kubectl path could not be accessed. Error=%s", err) + Logf("No kubectl in %s. Attempting a local build...", testStaticKubectlPath) + // Fall back to trying to build a local static kubectl + kubectlContainerPath := path.Join(testContext.RepoRoot, "/examples/kubectl-container/") + if _, err := os.Stat(path.Join(testContext.RepoRoot, "hack/build-go.sh")); err != nil { + Failf("Can't build static kubectl due to missing hack/build-go.sh. Error=%s", err) + } + By("Building a static kubectl for upload") + staticKubectlBuild := exec.Command("make", "-C", kubectlContainerPath) + if out, err := staticKubectlBuild.Output(); err != nil { + Failf("Unable to create static kubectl. Error=%s, Output=%q", err, out) + } + // Verify the static kubectl path + testStaticKubectlPath = path.Join(kubectlContainerPath, "kubectl") + _, err := os.Stat(testStaticKubectlPath) + if err != nil { + Failf("static kubectl path could not be found in %s. Error=%s", testStaticKubectlPath, err) + } } + By(fmt.Sprintf("Using the kubectl in %s", testStaticKubectlPath)) // Verify the kubeconfig path kubeConfigFilePath := testContext.KubeConfig From 7671f72fbdb52f41a5743eb78e117ca260d82fb4 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2015 19:20:48 -0500 Subject: [PATCH 11/13] Defer GinkgoRecover call in runKubectlWithTimeout to handle panic --- test/e2e/portforward.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/portforward.go b/test/e2e/portforward.go index 2acb59450bc2..36a9996a66b1 100644 --- a/test/e2e/portforward.go +++ b/test/e2e/portforward.go @@ -115,6 +115,7 @@ func runPortForward(ns, podName string, port int) (*exec.Cmd, int) { func runKubectlWithTimeout(timeout time.Duration, args ...string) string { logOutput := make(chan string) go func() { + defer GinkgoRecover() logOutput <- runKubectl(args...) }() select { From caf10e1339989f7e97a78cf053b5beb6e80881f5 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Sat, 7 Nov 2015 16:48:11 -0800 Subject: [PATCH 12/13] Dump kubectl logs of Ingress controller at on e2e failure --- test/e2e/ingress.go | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 0d378b95f26e..331d2b5da906 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -62,6 +62,9 @@ var ( expectedLBCreationTime = 7 * time.Minute expectedLBHealthCheckTime = 7 * time.Minute + // Name of the loadbalancer controller within the cluster addon + lbContainerName = "l7-lb-controller" + // On average it takes ~6 minutes for a single backend to come online. // We *don't* expect this poll to consistently take 15 minutes for every // Ingress as GCE is creating/checking backends in parallel, but at the @@ -201,6 +204,35 @@ func checkLeakedResources() error { return nil } +// run kubectl log on the L7 controller pod. +func kubectlLogLBController(c *client.Client) { + selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": "glbc"})) + podList, err := c.Pods(api.NamespaceAll).List(selector, fields.Everything()) + if err != nil { + Logf("Cannot log L7 controller output, error listing pods %v", err) + return + } + if len(podList.Items) == 0 { + Logf("Loadbalancer controller pod not found") + return + } + for _, p := range podList.Items { + Logf("\nLast 100 log lines of %v\n", p.Name) + Logf(runKubectl("logs", p.Name, "--namespace=kube-system", "-c", lbContainerName, "--tail=100")) + } +} + +// dumpDebugAndFail dumps verbose debug output before failing. +func dumpDebugAndFail(err string, ns string, c *client.Client) { + kubectlLogLBController(c) + Logf("\nOutput of kubectl describe ing:\n") + + // TODO: runKubectl will hard fail if kubectl fails, swap it out for + // something more befitting for a debug dumper. + Logf(runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))) + Failf(err) +} + var _ = Describe("GCE L7 LoadBalancer Controller", func() { // These variables are initialized after framework's beforeEach. var ns string @@ -268,6 +300,9 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { // Wait for the loadbalancer IP. start := time.Now() address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout) + if err != nil { + dumpDebugAndFail(fmt.Sprintf("Ingress failed to acquire an IP address within %v", lbPollTimeout), ns, client) + } Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online", address, ing.Name, time.Since(start))) @@ -295,8 +330,8 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { return true, nil }) if pollErr != nil { - Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", - lbPollTimeout, route, rules.Host, lastBody, pollErr) + dumpDebugAndFail(fmt.Sprintf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", + lbPollTimeout, route, rules.Host, lastBody, pollErr), ns, client) } rt := time.Since(GETStart) By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt)) @@ -310,12 +345,12 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { sort.Sort(timeSlice(creationTimes)) perc50 := creationTimes[len(creationTimes)/2] if perc50 > expectedLBCreationTime { - Failf("Average creation time is too high: %+v", creationTimes) + dumpDebugAndFail(fmt.Sprintf("Average creation time is too high: %+v", creationTimes), ns, client) } sort.Sort(timeSlice(responseTimes)) perc50 = responseTimes[len(responseTimes)/2] if perc50 > expectedLBHealthCheckTime { - Failf("Average startup time is too high: %+v", responseTimes) + dumpDebugAndFail(fmt.Sprintf("Average startup time is too high: %+v", responseTimes), ns, client) } }) }) From 632fd96b5cee53d6ac4ddf0251685edd8e4ea30b Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Sun, 8 Nov 2015 10:58:21 -0800 Subject: [PATCH 13/13] Refactor debug logging methods. --- test/e2e/examples.go | 86 ++++++++++++++++----------------- test/e2e/ingress.go | 47 ++++++++++-------- test/e2e/kubectl.go | 104 ++++++++++++++++++++-------------------- test/e2e/portforward.go | 2 +- test/e2e/util.go | 38 +++++++++------ 5 files changed, 148 insertions(+), 129 deletions(-) diff --git a/test/e2e/examples.go b/test/e2e/examples.go index 11f065f8d3dc..ee2f380c8dd4 100644 --- a/test/e2e/examples.go +++ b/test/e2e/examples.go @@ -85,7 +85,7 @@ var _ = Describe("Examples e2e", func() { expectedOnSentinel := "+monitor master" By("starting redis bootstrap") - runKubectl("create", "-f", bootstrapYaml, nsFlag) + runKubectlOrDie("create", "-f", bootstrapYaml, nsFlag) err := waitForPodRunningInNamespace(c, bootstrapPodName, ns) Expect(err).NotTo(HaveOccurred()) @@ -95,13 +95,13 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("setting up services and controllers") - runKubectl("create", "-f", sentinelServiceYaml, nsFlag) - runKubectl("create", "-f", sentinelControllerYaml, nsFlag) - runKubectl("create", "-f", controllerYaml, nsFlag) + runKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", controllerYaml, nsFlag) By("scaling up the deployment") - runKubectl("scale", "rc", redisRC, "--replicas=3", nsFlag) - runKubectl("scale", "rc", sentinelRC, "--replicas=3", nsFlag) + runKubectlOrDie("scale", "rc", redisRC, "--replicas=3", nsFlag) + runKubectlOrDie("scale", "rc", sentinelRC, "--replicas=3", nsFlag) By("checking up the services") checkAllLogs := func() { @@ -121,7 +121,7 @@ var _ = Describe("Examples e2e", func() { checkAllLogs() By("turning down bootstrap") - runKubectl("delete", "-f", bootstrapYaml, nsFlag) + runKubectlOrDie("delete", "-f", bootstrapYaml, nsFlag) err = waitForRCPodToDisappear(c, ns, redisRC, bootstrapPodName) Expect(err).NotTo(HaveOccurred()) By("waiting for the new master election") @@ -142,8 +142,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting rabbitmq") - runKubectl("create", "-f", rabbitmqServiceYaml, nsFlag) - runKubectl("create", "-f", rabbitmqControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", rabbitmqServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", rabbitmqControllerYaml, nsFlag) forEachPod(c, ns, "component", "rabbitmq", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) @@ -152,15 +152,15 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting celery") - runKubectl("create", "-f", celeryControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", celeryControllerYaml, nsFlag) forEachPod(c, ns, "component", "celery", func(pod api.Pod) { _, err := lookForStringInFile(ns, pod.Name, "celery", "/data/celery.log", " ready.", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) }) By("starting flower") - runKubectl("create", "-f", flowerServiceYaml, nsFlag) - runKubectl("create", "-f", flowerControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", flowerServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", flowerControllerYaml, nsFlag) forEachPod(c, ns, "component", "flower", func(pod api.Pod) { // Do nothing. just wait for it to be up and running. }) @@ -184,9 +184,9 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting master") - runKubectl("create", "-f", serviceJson, nsFlag) - runKubectl("create", "-f", masterJson, nsFlag) - runKubectl("create", "-f", driverJson, nsFlag) + runKubectlOrDie("create", "-f", serviceJson, nsFlag) + runKubectlOrDie("create", "-f", masterJson, nsFlag) + runKubectlOrDie("create", "-f", driverJson, nsFlag) err := waitForPodRunningInNamespace(c, "spark-master", ns) Expect(err).NotTo(HaveOccurred()) _, err = lookForStringInLog(ns, "spark-master", "spark-master", "Starting Spark master at", serverStartTimeout) @@ -199,7 +199,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting workers") - runKubectl("create", "-f", workerControllerJson, nsFlag) + runKubectlOrDie("create", "-f", workerControllerJson, nsFlag) ScaleRC(c, ns, "spark-worker-controller", 2, true) forEachPod(c, ns, "name", "spark-worker", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "spark-worker", "Successfully registered with master", serverStartTimeout) @@ -219,8 +219,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting service and pod") - runKubectl("create", "-f", serviceYaml, nsFlag) - runKubectl("create", "-f", podYaml, nsFlag) + runKubectlOrDie("create", "-f", serviceYaml, nsFlag) + runKubectlOrDie("create", "-f", podYaml, nsFlag) err := waitForPodRunningInNamespace(c, "cassandra", ns) Expect(err).NotTo(HaveOccurred()) @@ -231,7 +231,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("create and scale rc") - runKubectl("create", "-f", controllerYaml, nsFlag) + runKubectlOrDie("create", "-f", controllerYaml, nsFlag) err = ScaleRC(c, ns, "cassandra", 2, true) Expect(err).NotTo(HaveOccurred()) forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) { @@ -241,7 +241,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) }) - output := runKubectl("exec", "cassandra", nsFlag, "--", "nodetool", "status") + output := runKubectlOrDie("exec", "cassandra", nsFlag, "--", "nodetool", "status") forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) { if !strings.Contains(output, pod.Status.PodIP) { Failf("Pod ip %s not found in nodetool status", pod.Status.PodIP) @@ -264,8 +264,8 @@ var _ = Describe("Examples e2e", func() { zookeeperPod := "zookeeper" By("starting Zookeeper") - runKubectl("create", "-f", zookeeperPodJson, nsFlag) - runKubectl("create", "-f", zookeeperServiceJson, nsFlag) + runKubectlOrDie("create", "-f", zookeeperPodJson, nsFlag) + runKubectlOrDie("create", "-f", zookeeperServiceJson, nsFlag) err := waitForPodRunningInNamespace(c, zookeeperPod, ns) Expect(err).NotTo(HaveOccurred()) @@ -276,8 +276,8 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting Nimbus") - runKubectl("create", "-f", nimbusPodJson, nsFlag) - runKubectl("create", "-f", nimbusServiceJson, nsFlag) + runKubectlOrDie("create", "-f", nimbusPodJson, nsFlag) + runKubectlOrDie("create", "-f", nimbusServiceJson, nsFlag) err = waitForPodRunningInNamespace(c, "nimbus", ns) Expect(err).NotTo(HaveOccurred()) @@ -285,7 +285,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting workers") - runKubectl("create", "-f", workerControllerJson, nsFlag) + runKubectlOrDie("create", "-f", workerControllerJson, nsFlag) forEachPod(c, ns, "name", "storm-worker", func(pod api.Pod) { //do nothing, just wait for the pod to be running }) @@ -299,7 +299,7 @@ var _ = Describe("Examples e2e", func() { By("checking if Nimbus responds to requests") lookForString("No topologies running.", time.Minute, func() string { - return runKubectl("exec", "nimbus", nsFlag, "--", "bin/storm", "list") + return runKubectlOrDie("exec", "nimbus", nsFlag, "--", "bin/storm", "list") }) }) }) @@ -313,8 +313,8 @@ var _ = Describe("Examples e2e", func() { httpYaml := mkpath("http-liveness.yaml") nsFlag := fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", execYaml, nsFlag) - runKubectl("create", "-f", httpYaml, nsFlag) + runKubectlOrDie("create", "-f", execYaml, nsFlag) + runKubectlOrDie("create", "-f", httpYaml, nsFlag) checkRestart := func(podName string, timeout time.Duration) { err := waitForPodRunningInNamespace(c, podName, ns) Expect(err).NotTo(HaveOccurred()) @@ -346,8 +346,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("creating secret and pod") - runKubectl("create", "-f", secretYaml, nsFlag) - runKubectl("create", "-f", podYaml, nsFlag) + runKubectlOrDie("create", "-f", secretYaml, nsFlag) + runKubectlOrDie("create", "-f", podYaml, nsFlag) By("checking if secret was read correctly") _, err := lookForStringInLog(ns, "secret-test-pod", "test-container", "value-1", serverStartTimeout) @@ -365,7 +365,7 @@ var _ = Describe("Examples e2e", func() { podName := "dapi-test-pod" By("creating the pod") - runKubectl("create", "-f", podYaml, nsFlag) + runKubectlOrDie("create", "-f", podYaml, nsFlag) By("checking if name and namespace were passed correctly") _, err := lookForStringInLog(ns, podName, "test-container", fmt.Sprintf("POD_NAMESPACE=%v", ns), serverStartTimeout) @@ -387,8 +387,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting rethinkdb") - runKubectl("create", "-f", driverServiceYaml, nsFlag) - runKubectl("create", "-f", rethinkDbControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", driverServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", rethinkDbControllerYaml, nsFlag) checkDbInstances := func() { forEachPod(c, ns, "db", "rethinkdb", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "rethinkdb", "Server ready", serverStartTimeout) @@ -404,8 +404,8 @@ var _ = Describe("Examples e2e", func() { checkDbInstances() By("starting admin") - runKubectl("create", "-f", adminServiceYaml, nsFlag) - runKubectl("create", "-f", adminPodYaml, nsFlag) + runKubectlOrDie("create", "-f", adminServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", adminPodYaml, nsFlag) err = waitForPodRunningInNamespace(c, "rethinkdb-admin", ns) Expect(err).NotTo(HaveOccurred()) checkDbInstances() @@ -427,8 +427,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting hazelcast") - runKubectl("create", "-f", serviceYaml, nsFlag) - runKubectl("create", "-f", controllerYaml, nsFlag) + runKubectlOrDie("create", "-f", serviceYaml, nsFlag) + runKubectlOrDie("create", "-f", controllerYaml, nsFlag) forEachPod(c, ns, "name", "hazelcast", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "hazelcast", "Members [1]", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) @@ -483,11 +483,11 @@ var _ = Describe("Examples e2e", func() { } for _, ns := range namespaces { - runKubectl("create", "-f", backendRcYaml, getNsCmdFlag(ns)) + runKubectlOrDie("create", "-f", backendRcYaml, getNsCmdFlag(ns)) } for _, ns := range namespaces { - runKubectl("create", "-f", backendSvcYaml, getNsCmdFlag(ns)) + runKubectlOrDie("create", "-f", backendSvcYaml, getNsCmdFlag(ns)) } // wait for objects @@ -533,7 +533,7 @@ var _ = Describe("Examples e2e", func() { // create a pod in each namespace for _, ns := range namespaces { - newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).exec() + newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).execOrDie() } // wait until the pods have been scheduler, i.e. are not Pending anymore. Remember @@ -608,13 +608,13 @@ func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func func lookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) { return lookForString(expectedString, timeout, func() string { - return runKubectl("log", podName, container, fmt.Sprintf("--namespace=%v", ns)) + return runKubectlOrDie("log", podName, container, fmt.Sprintf("--namespace=%v", ns)) }) } func lookForStringInFile(ns, podName, container, file, expectedString string, timeout time.Duration) (result string, err error) { return lookForString(expectedString, timeout, func() string { - return runKubectl("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file) + return runKubectlOrDie("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file) }) } @@ -623,7 +623,7 @@ func lookForStringInPodExec(ns, podName string, command []string, expectedString // use the first container args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns), "--"} args = append(args, command...) - return runKubectl(args...) + return runKubectlOrDie(args...) }) } diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 331d2b5da906..6acf33833c16 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -65,6 +65,10 @@ var ( // Name of the loadbalancer controller within the cluster addon lbContainerName = "l7-lb-controller" + // If set, the test tries to perform an HTTP GET on each url endpoint of + // the Ingress. Only set to false to short-circuit test runs in debugging. + verifyHTTPGET = true + // On average it takes ~6 minutes for a single backend to come online. // We *don't* expect this poll to consistently take 15 minutes for every // Ingress as GCE is creating/checking backends in parallel, but at the @@ -173,7 +177,9 @@ func createApp(c *client.Client, ns string, i int) { // gcloudUnmarshal unmarshals json output of gcloud into given out interface. func gcloudUnmarshal(resource, regex string, out interface{}) { output, err := exec.Command("gcloud", "compute", resource, "list", - fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput() + fmt.Sprintf("--regex=%v", regex), + fmt.Sprintf("--project=%v", testContext.CloudConfig.ProjectID), + "-q", "--format=json").CombinedOutput() if err != nil { Failf("Error unmarshalling gcloud output: %v", err) } @@ -204,7 +210,7 @@ func checkLeakedResources() error { return nil } -// run kubectl log on the L7 controller pod. +// kubectlLogLBController logs kubectl debug output for the L7 controller pod. func kubectlLogLBController(c *client.Client) { selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": "glbc"})) podList, err := c.Pods(api.NamespaceAll).List(selector, fields.Everything()) @@ -218,21 +224,11 @@ func kubectlLogLBController(c *client.Client) { } for _, p := range podList.Items { Logf("\nLast 100 log lines of %v\n", p.Name) - Logf(runKubectl("logs", p.Name, "--namespace=kube-system", "-c", lbContainerName, "--tail=100")) + l, _ := runKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", api.NamespaceSystem), "-c", lbContainerName, "--tail=100") + Logf(l) } } -// dumpDebugAndFail dumps verbose debug output before failing. -func dumpDebugAndFail(err string, ns string, c *client.Client) { - kubectlLogLBController(c) - Logf("\nOutput of kubectl describe ing:\n") - - // TODO: runKubectl will hard fail if kubectl fails, swap it out for - // something more befitting for a debug dumper. - Logf(runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))) - Failf(err) -} - var _ = Describe("GCE L7 LoadBalancer Controller", func() { // These variables are initialized after framework's beforeEach. var ns string @@ -254,6 +250,12 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { }) AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + kubectlLogLBController(client) + Logf("\nOutput of kubectl describe ing:\n") + desc, _ := runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns)) + Logf(desc) + } framework.afterEach() err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) { if err := checkLeakedResources(); err != nil { @@ -301,13 +303,17 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { start := time.Now() address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout) if err != nil { - dumpDebugAndFail(fmt.Sprintf("Ingress failed to acquire an IP address within %v", lbPollTimeout), ns, client) + Failf("Ingress failed to acquire an IP address within %v", lbPollTimeout) } Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online", address, ing.Name, time.Since(start))) creationTimes = append(creationTimes, time.Since(start)) + if !verifyHTTPGET { + continue + } + // Check that all rules respond to a simple GET. for _, rules := range ing.Spec.Rules { // As of Kubernetes 1.1 we only support HTTP Ingress. @@ -330,8 +336,8 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { return true, nil }) if pollErr != nil { - dumpDebugAndFail(fmt.Sprintf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", - lbPollTimeout, route, rules.Host, lastBody, pollErr), ns, client) + Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", + lbPollTimeout, route, rules.Host, lastBody, pollErr) } rt := time.Since(GETStart) By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt)) @@ -345,12 +351,15 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { sort.Sort(timeSlice(creationTimes)) perc50 := creationTimes[len(creationTimes)/2] if perc50 > expectedLBCreationTime { - dumpDebugAndFail(fmt.Sprintf("Average creation time is too high: %+v", creationTimes), ns, client) + Failf("Average creation time is too high: %+v", creationTimes) + } + if !verifyHTTPGET { + return } sort.Sort(timeSlice(responseTimes)) perc50 = responseTimes[len(responseTimes)/2] if perc50 > expectedLBHealthCheckTime { - dumpDebugAndFail(fmt.Sprintf("Average startup time is too high: %+v", responseTimes), ns, client) + Failf("Average startup time is too high: %+v", responseTimes) } }) }) diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index 2d86da7763c9..d8f50a5d7f7c 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -106,7 +106,7 @@ var _ = Describe("Kubectl client", func() { defer cleanup(nautilusPath, ns, updateDemoSelector) By("creating a replication controller") - runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) }) @@ -114,22 +114,22 @@ var _ = Describe("Kubectl client", func() { defer cleanup(nautilusPath, ns, updateDemoSelector) By("creating a replication controller") - runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) By("scaling down the replication controller") - runKubectl("scale", "rc", "update-demo-nautilus", "--replicas=1", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("scale", "rc", "update-demo-nautilus", "--replicas=1", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 1, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) By("scaling up the replication controller") - runKubectl("scale", "rc", "update-demo-nautilus", "--replicas=2", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("scale", "rc", "update-demo-nautilus", "--replicas=2", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) }) It("should do a rolling update of a replication controller [Conformance]", func() { By("creating the initial replication controller") - runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) By("rolling-update to new replication controller") - runKubectl("rolling-update", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("rolling-update", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, kittenImage, 2, "update-demo", updateDemoSelector, getUDData("kitten.jpg", ns), ns) // Everything will hopefully be cleaned up when the namespace is deleted. }) @@ -146,7 +146,7 @@ var _ = Describe("Kubectl client", func() { defer cleanup(guestbookPath, ns, frontendSelector, redisMasterSelector, redisSlaveSelector) By("creating all guestbook components") - runKubectl("create", "-f", guestbookPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", guestbookPath, fmt.Sprintf("--namespace=%v", ns)) By("validating guestbook app") validateGuestbookApp(c, ns) @@ -159,7 +159,7 @@ var _ = Describe("Kubectl client", func() { BeforeEach(func() { podPath = filepath.Join(testContext.RepoRoot, "docs/user-guide/pod.yaml") By("creating the pod") - runKubectl("create", "-f", podPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", podPath, fmt.Sprintf("--namespace=%v", ns)) checkPodsRunningReady(c, ns, []string{simplePodName}, podStartTimeout) }) AfterEach(func() { @@ -168,7 +168,7 @@ var _ = Describe("Kubectl client", func() { It("should support exec", func() { By("executing a command in the container") - execOutput := runKubectl("exec", fmt.Sprintf("--namespace=%v", ns), simplePodName, "echo", "running", "in", "container") + execOutput := runKubectlOrDie("exec", fmt.Sprintf("--namespace=%v", ns), simplePodName, "echo", "running", "in", "container") if e, a := "running in container", execOutput; e != a { Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a) } @@ -176,7 +176,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command in the container with noninteractive stdin") execOutput = newKubectlCommand("exec", fmt.Sprintf("--namespace=%v", ns), "-i", simplePodName, "cat"). withStdinData("abcd1234"). - exec() + execOrDie() if e, a := "abcd1234", execOutput; e != a { Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a) } @@ -192,7 +192,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command in the container with pseudo-interactive stdin") execOutput = newKubectlCommand("exec", fmt.Sprintf("--namespace=%v", ns), "-i", simplePodName, "bash"). withStdinReader(r). - exec() + execOrDie() if e, a := "hi", execOutput; e != a { Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a) } @@ -253,7 +253,7 @@ var _ = Describe("Kubectl client", func() { } // start exec-proxy-tester container netexecPodPath := filepath.Join(testContext.RepoRoot, "test/images/netexec/pod.yaml") - runKubectl("create", "-f", netexecPodPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", netexecPodPath, fmt.Sprintf("--namespace=%v", ns)) checkPodsRunningReady(c, ns, []string{netexecContainer}, podStartTimeout) // Clean up defer cleanup(netexecPodPath, ns, netexecPodSelector) @@ -323,7 +323,7 @@ var _ = Describe("Kubectl client", func() { By("Running kubectl in netexec via an HTTP proxy using " + proxyVar) // start the proxy container goproxyPodPath := filepath.Join(testContext.RepoRoot, "test/images/goproxy/pod.yaml") - runKubectl("create", "-f", goproxyPodPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", goproxyPodPath, fmt.Sprintf("--namespace=%v", ns)) checkPodsRunningReady(c, ns, []string{goproxyContainer}, podStartTimeout) // get the proxy address @@ -360,7 +360,7 @@ var _ = Describe("Kubectl client", func() { // Verify the proxy server logs saw the connection expectedProxyLog := fmt.Sprintf("Accepting CONNECT to %s", strings.TrimRight(strings.TrimLeft(testContext.Host, "https://"), "/api")) - proxyLog := runKubectl("log", "goproxy", fmt.Sprintf("--namespace=%v", ns)) + proxyLog := runKubectlOrDie("log", "goproxy", fmt.Sprintf("--namespace=%v", ns)) if !strings.Contains(proxyLog, expectedProxyLog) { Failf("Missing expected log result on proxy server for %s. Expected: %q, got %q", proxyVar, expectedProxyLog, proxyLog) @@ -376,7 +376,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command with run and attach with stdin") runOutput := newKubectlCommand(nsFlag, "run", "run-test", "--image=busybox", "--restart=Never", "--attach=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'"). withStdinData("abcd1234"). - exec() + execOrDie() Expect(runOutput).To(ContainSubstring("abcd1234")) Expect(runOutput).To(ContainSubstring("stdin closed")) Expect(c.Pods(ns).Delete("run-test", api.NewDeleteOptions(0))).To(BeNil()) @@ -384,7 +384,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command with run and attach without stdin") runOutput = newKubectlCommand(fmt.Sprintf("--namespace=%v", ns), "run", "run-test-2", "--image=busybox", "--restart=Never", "--attach=true", "--leave-stdin-open=true", "--", "sh", "-c", "cat && echo 'stdin closed'"). withStdinData("abcd1234"). - exec() + execOrDie() Expect(runOutput).ToNot(ContainSubstring("abcd1234")) Expect(runOutput).To(ContainSubstring("stdin closed")) Expect(c.Pods(ns).Delete("run-test-2", api.NewDeleteOptions(0))).To(BeNil()) @@ -392,7 +392,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command with run and attach with stdin with open stdin should remain running") runOutput = newKubectlCommand(nsFlag, "run", "run-test-3", "--image=busybox", "--restart=Never", "--attach=true", "--leave-stdin-open=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'"). withStdinData("abcd1234\n"). - exec() + execOrDie() Expect(runOutput).ToNot(ContainSubstring("stdin closed")) if !checkPodsRunningReady(c, ns, []string{"run-test-3"}, time.Minute) { Failf("Pod %q should still be running", "run-test-3") @@ -404,7 +404,7 @@ var _ = Describe("Kubectl client", func() { if !checkPodsRunningReady(c, ns, []string{"run-test-3"}, 1*time.Second) { Failf("Pod %q should still be running", "run-test-3") } - logOutput := runKubectl(nsFlag, "logs", "run-test-3") + logOutput := runKubectlOrDie(nsFlag, "logs", "run-test-3") Expect(logOutput).ToNot(ContainSubstring("stdin closed")) return strings.Contains(logOutput, "abcd1234"), nil }) @@ -437,7 +437,7 @@ var _ = Describe("Kubectl client", func() { Describe("Kubectl api-versions", func() { It("should check if v1 is in available api versions [Conformance]", func() { By("validating api verions") - output := runKubectl("api-versions") + output := runKubectlOrDie("api-versions") if !strings.Contains(output, "v1") { Failf("No v1 in kubectl api-versions") } @@ -452,12 +452,12 @@ var _ = Describe("Kubectl client", func() { controllerJson := mkpath("redis-master-controller.json") nsFlag := fmt.Sprintf("--namespace=%v", ns) By("creating Redis RC") - runKubectl("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) By("applying a modified configuration") stdin := modifyReplicationControllerConfiguration(controllerJson) newKubectlCommand("apply", "-f", "-", nsFlag). withStdinReader(stdin). - exec() + execOrDie() By("checking the result") forEachReplicationController(c, ns, "app", "redis", validateReplicationControllerConfiguration) }) @@ -466,7 +466,7 @@ var _ = Describe("Kubectl client", func() { Describe("Kubectl cluster-info", func() { It("should check if Kubernetes master services is included in cluster-info [Conformance]", func() { By("validating cluster-info") - output := runKubectl("cluster-info") + output := runKubectlOrDie("cluster-info") // Can't check exact strings due to terminal control commands (colors) requiredItems := []string{"Kubernetes master", "is running at"} if providerIs("gce", "gke") { @@ -489,12 +489,12 @@ var _ = Describe("Kubectl client", func() { serviceJson := mkpath("redis-master-service.json") nsFlag := fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", controllerJson, nsFlag) - runKubectl("create", "-f", serviceJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", serviceJson, nsFlag) // Pod forEachPod(c, ns, "app", "redis", func(pod api.Pod) { - output := runKubectl("describe", "pod", pod.Name, nsFlag) + output := runKubectlOrDie("describe", "pod", pod.Name, nsFlag) requiredStrings := [][]string{ {"Name:", "redis-master-"}, {"Namespace:", ns}, @@ -510,7 +510,7 @@ var _ = Describe("Kubectl client", func() { }) // Rc - output := runKubectl("describe", "rc", "redis-master", nsFlag) + output := runKubectlOrDie("describe", "rc", "redis-master", nsFlag) requiredStrings := [][]string{ {"Name:", "redis-master"}, {"Namespace:", ns}, @@ -523,7 +523,7 @@ var _ = Describe("Kubectl client", func() { checkOutput(output, requiredStrings) // Service - output = runKubectl("describe", "service", "redis-master", nsFlag) + output = runKubectlOrDie("describe", "service", "redis-master", nsFlag) requiredStrings = [][]string{ {"Name:", "redis-master"}, {"Namespace:", ns}, @@ -540,7 +540,7 @@ var _ = Describe("Kubectl client", func() { nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) Expect(err).NotTo(HaveOccurred()) node := nodes.Items[0] - output = runKubectl("describe", "node", node.Name) + output = runKubectlOrDie("describe", "node", node.Name) requiredStrings = [][]string{ {"Name:", node.Name}, {"Labels:"}, @@ -559,7 +559,7 @@ var _ = Describe("Kubectl client", func() { checkOutput(output, requiredStrings) // Namespace - output = runKubectl("describe", "namespace", ns) + output = runKubectlOrDie("describe", "namespace", ns) requiredStrings = [][]string{ {"Name:", ns}, {"Labels:"}, @@ -581,7 +581,7 @@ var _ = Describe("Kubectl client", func() { redisPort := 6379 By("creating Redis RC") - runKubectl("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) forEachPod(c, ns, "app", "redis", func(pod api.Pod) { lookForStringInLog(ns, pod.Name, "redis-master", "The server is now ready to accept connections", podStartTimeout) }) @@ -629,12 +629,12 @@ var _ = Describe("Kubectl client", func() { } By("exposing RC") - runKubectl("expose", "rc", "redis-master", "--name=rm2", "--port=1234", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) + runKubectlOrDie("expose", "rc", "redis-master", "--name=rm2", "--port=1234", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) waitForService(c, ns, "rm2", true, poll, serviceStartTimeout) validateService("rm2", 1234, serviceStartTimeout) By("exposing service") - runKubectl("expose", "service", "rm2", "--name=rm3", "--port=2345", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) + runKubectlOrDie("expose", "service", "rm2", "--name=rm3", "--port=2345", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) waitForService(c, ns, "rm3", true, poll, serviceStartTimeout) validateService("rm3", 2345, serviceStartTimeout) }) @@ -647,7 +647,7 @@ var _ = Describe("Kubectl client", func() { podPath = filepath.Join(testContext.RepoRoot, "docs/user-guide/pod.yaml") By("creating the pod") nsFlag = fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", podPath, nsFlag) + runKubectlOrDie("create", "-f", podPath, nsFlag) checkPodsRunningReady(c, ns, []string{simplePodName}, podStartTimeout) }) AfterEach(func() { @@ -659,17 +659,17 @@ var _ = Describe("Kubectl client", func() { labelValue := "testing-label-value" By("adding the label " + labelName + " with value " + labelValue + " to a pod") - runKubectl("label", "pods", simplePodName, labelName+"="+labelValue, nsFlag) + runKubectlOrDie("label", "pods", simplePodName, labelName+"="+labelValue, nsFlag) By("verifying the pod has the label " + labelName + " with the value " + labelValue) - output := runKubectl("get", "pod", simplePodName, "-L", labelName, nsFlag) + output := runKubectlOrDie("get", "pod", simplePodName, "-L", labelName, nsFlag) if !strings.Contains(output, labelValue) { Failf("Failed updating label " + labelName + " to the pod " + simplePodName) } By("removing the label " + labelName + " of a pod") - runKubectl("label", "pods", simplePodName, labelName+"-", nsFlag) + runKubectlOrDie("label", "pods", simplePodName, labelName+"-", nsFlag) By("verifying the pod doesn't have the label " + labelName) - output = runKubectl("get", "pod", simplePodName, "-L", labelName, nsFlag) + output = runKubectlOrDie("get", "pod", simplePodName, "-L", labelName, nsFlag) if strings.Contains(output, labelValue) { Failf("Failed removing label " + labelName + " of the pod " + simplePodName) } @@ -687,7 +687,7 @@ var _ = Describe("Kubectl client", func() { rcPath = mkpath("redis-master-controller.json") By("creating an rc") nsFlag = fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", rcPath, nsFlag) + runKubectlOrDie("create", "-f", rcPath, nsFlag) }) AfterEach(func() { cleanup(rcPath, ns, simplePodSelector) @@ -700,17 +700,17 @@ var _ = Describe("Kubectl client", func() { Expect(err).NotTo(HaveOccurred()) By("limiting log lines") - out := runKubectl("log", pod.Name, containerName, nsFlag, "--tail=1") + out := runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--tail=1") Expect(len(out)).NotTo(BeZero()) Expect(len(strings.Split(out, "\n"))).To(Equal(1)) By("limiting log bytes") - out = runKubectl("log", pod.Name, containerName, nsFlag, "--limit-bytes=1") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--limit-bytes=1") Expect(len(strings.Split(out, "\n"))).To(Equal(1)) Expect(len(out)).To(Equal(1)) By("exposing timestamps") - out = runKubectl("log", pod.Name, containerName, nsFlag, "--tail=1", "--timestamps") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--tail=1", "--timestamps") lines := strings.Split(out, "\n") Expect(len(lines)).To(Equal(1)) words := strings.Split(lines[0], " ") @@ -723,9 +723,9 @@ var _ = Describe("Kubectl client", func() { By("restricting to a time range") time.Sleep(1500 * time.Millisecond) // ensure that startup logs on the node are seen as older than 1s - out = runKubectl("log", pod.Name, containerName, nsFlag, "--since=1s") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=1s") recent := len(strings.Split(out, "\n")) - out = runKubectl("log", pod.Name, containerName, nsFlag, "--since=24h") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=24h") older := len(strings.Split(out, "\n")) Expect(recent).To(BeNumerically("<", older)) }) @@ -740,10 +740,10 @@ var _ = Describe("Kubectl client", func() { controllerJson := mkpath("redis-master-controller.json") nsFlag := fmt.Sprintf("--namespace=%v", ns) By("creating Redis RC") - runKubectl("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) By("patching all pods") forEachPod(c, ns, "app", "redis", func(pod api.Pod) { - runKubectl("patch", "pod", pod.Name, nsFlag, "-p", "{\"metadata\":{\"annotations\":{\"x\":\"y\"}}}") + runKubectlOrDie("patch", "pod", pod.Name, nsFlag, "-p", "{\"metadata\":{\"annotations\":{\"x\":\"y\"}}}") }) By("checking annotations") @@ -763,7 +763,7 @@ var _ = Describe("Kubectl client", func() { Describe("Kubectl version", func() { It("should check is all data is printed [Conformance]", func() { - version := runKubectl("version") + version := runKubectlOrDie("version") requiredItems := []string{"Client Version:", "Server Version:", "Major:", "Minor:", "GitCommit:"} for _, item := range requiredItems { if !strings.Contains(version, item) { @@ -783,14 +783,14 @@ var _ = Describe("Kubectl client", func() { }) AfterEach(func() { - runKubectl("stop", "rc", rcName, nsFlag) + runKubectlOrDie("stop", "rc", rcName, nsFlag) }) It("should create an rc from an image [Conformance]", func() { image := "nginx" By("running the image " + image) - runKubectl("run", rcName, "--image="+image, nsFlag) + runKubectlOrDie("run", rcName, "--image="+image, nsFlag) By("verifying the rc " + rcName + " was created") rc, err := c.ReplicationControllers(ns).Get(rcName) if err != nil { @@ -809,7 +809,7 @@ var _ = Describe("Kubectl client", func() { } pods := podlist.Items if pods == nil || len(pods) != 1 || len(pods[0].Spec.Containers) != 1 || pods[0].Spec.Containers[0].Image != image { - runKubectl("get", "pods", "-L", "run", nsFlag) + runKubectlOrDie("get", "pods", "-L", "run", nsFlag) Failf("Failed creating 1 pod with expected image %s. Number of pods = %v", image, len(pods)) } }) @@ -826,14 +826,14 @@ var _ = Describe("Kubectl client", func() { }) AfterEach(func() { - runKubectl("stop", "pods", podName, nsFlag) + runKubectlOrDie("stop", "pods", podName, nsFlag) }) It("should create a pod from an image when restart is OnFailure [Conformance]", func() { image := "nginx" By("running the image " + image) - runKubectl("run", podName, "--restart=OnFailure", "--image="+image, nsFlag) + runKubectlOrDie("run", podName, "--restart=OnFailure", "--image="+image, nsFlag) By("verifying the pod " + podName + " was created") pod, err := c.Pods(ns).Get(podName) if err != nil { @@ -852,7 +852,7 @@ var _ = Describe("Kubectl client", func() { image := "nginx" By("running the image " + image) - runKubectl("run", podName, "--restart=Never", "--image="+image, nsFlag) + runKubectlOrDie("run", podName, "--restart=Never", "--image="+image, nsFlag) By("verifying the pod " + podName + " was created") pod, err := c.Pods(ns).Get(podName) if err != nil { diff --git a/test/e2e/portforward.go b/test/e2e/portforward.go index 36a9996a66b1..582cc179814a 100644 --- a/test/e2e/portforward.go +++ b/test/e2e/portforward.go @@ -116,7 +116,7 @@ func runKubectlWithTimeout(timeout time.Duration, args ...string) string { logOutput := make(chan string) go func() { defer GinkgoRecover() - logOutput <- runKubectl(args...) + logOutput <- runKubectlOrDie(args...) }() select { case <-time.After(timeout): diff --git a/test/e2e/util.go b/test/e2e/util.go index 343a7fea4b24..1d35ac695a1a 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -892,14 +892,14 @@ func cleanup(filePath string, ns string, selectors ...string) { if ns != "" { nsArg = fmt.Sprintf("--namespace=%s", ns) } - runKubectl("stop", "--grace-period=0", "-f", filePath, nsArg) + runKubectlOrDie("stop", "--grace-period=0", "-f", filePath, nsArg) for _, selector := range selectors { - resources := runKubectl("get", "rc,svc", "-l", selector, "--no-headers", nsArg) + resources := runKubectlOrDie("get", "rc,svc", "-l", selector, "--no-headers", nsArg) if resources != "" { Failf("Resources left running after stop:\n%s", resources) } - pods := runKubectl("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}") + pods := runKubectlOrDie("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}") if pods != "" { Failf("Pods left unterminated after stop:\n%s", pods) } @@ -933,7 +933,7 @@ func validateController(c *client.Client, containerImage string, replicas int, c By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector waitLoop: for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { - getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "--api-version=v1", "-l", testname, fmt.Sprintf("--namespace=%v", ns)) + getPodsOutput := runKubectlOrDie("get", "pods", "-o", "template", getPodsTemplate, "--api-version=v1", "-l", testname, fmt.Sprintf("--namespace=%v", ns)) pods := strings.Fields(getPodsOutput) if numPods := len(pods); numPods != replicas { By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", testname, replicas, numPods)) @@ -941,13 +941,13 @@ waitLoop: } var runningPods []string for _, podID := range pods { - running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) + running := runKubectlOrDie("get", "pods", podID, "-o", "template", getContainerStateTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) if running != "true" { Logf("%s is created but not running", podID) continue waitLoop } - currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) + currentImage := runKubectlOrDie("get", "pods", podID, "-o", "template", getImageTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) if currentImage != containerImage { Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, containerImage, currentImage) continue waitLoop @@ -1028,29 +1028,39 @@ func (b kubectlBuilder) withStdinReader(reader io.Reader) *kubectlBuilder { return &b } -func (b kubectlBuilder) exec() string { +func (b kubectlBuilder) execOrDie() string { + str, err := b.exec() + Expect(err).NotTo(HaveOccurred()) + return str +} + +func (b kubectlBuilder) exec() (string, error) { var stdout, stderr bytes.Buffer cmd := b.cmd cmd.Stdout, cmd.Stderr = &stdout, &stderr Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately if err := cmd.Run(); err != nil { - Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) - return "" + return "", fmt.Errorf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) } Logf(stdout.String()) // TODO: trimspace should be unnecessary after switching to use kubectl binary directly - return strings.TrimSpace(stdout.String()) + return strings.TrimSpace(stdout.String()), nil +} + +// runKubectlOrDie is a convenience wrapper over kubectlBuilder +func runKubectlOrDie(args ...string) string { + return newKubectlCommand(args...).execOrDie() } // runKubectl is a convenience wrapper over kubectlBuilder -func runKubectl(args ...string) string { +func runKubectl(args ...string) (string, error) { return newKubectlCommand(args...).exec() } -// runKubectlInput is a convenience wrapper over kubectlBuilder that takes input to stdin -func runKubectlInput(data string, args ...string) string { - return newKubectlCommand(args...).withStdinData(data).exec() +// runKubectlOrDieInput is a convenience wrapper over kubectlBuilder that takes input to stdin +func runKubectlOrDieInput(data string, args ...string) string { + return newKubectlCommand(args...).withStdinData(data).execOrDie() } func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) {