From 7a073f54bed21b35bd79001cc9828b5e6edc52d7 Mon Sep 17 00:00:00 2001 From: Stephen Date: Sun, 13 Oct 2013 23:48:15 +0100 Subject: [PATCH] Export Connection.Transport for use on Appengine Some environments, such as Google Appengine, have restrictions on sockets. An http transport can be used with Connection to interface to the custom backend: import ( "appengine/urlfetch" "fmt" "github.com/ncw/swift" ) func handler(w http.ResponseWriter, r *http.Request) { ctx := appengine.NewContext(r) c := swift.Connection{ UserName: "user", ApiKey: "key", AuthUrl: "auth_url", Transport: urlfetch.Transport{Context: ctx}, } err := c.Authenticate() if err != nil { panic(err) } fmt.Fprintf(w, "Authenticate OK") } Fixes #5 Original fix by @vmihailenko --- compatibility_1_0.go | 2 +- compatibility_1_1.go | 8 +++-- swift.go | 71 +++++++++++++++++++++++++++++++------------- swift_test.go | 30 ++++++++++++++++++- 4 files changed, 87 insertions(+), 24 deletions(-) diff --git a/compatibility_1_0.go b/compatibility_1_0.go index 0df4819ff..7b69a757a 100644 --- a/compatibility_1_0.go +++ b/compatibility_1_0.go @@ -11,7 +11,7 @@ import ( ) // Cancel the request - doesn't work under < go 1.1 -func cancelRequest(tr *http.Transport, req *http.Request) { +func cancelRequest(transport http.RoundTripper, req *http.Request) { log.Printf("Tried to cancel a request but couldn't - recompile with go 1.1") } diff --git a/compatibility_1_1.go b/compatibility_1_1.go index f53ae691e..a4f9c3ab2 100644 --- a/compatibility_1_1.go +++ b/compatibility_1_1.go @@ -10,8 +10,12 @@ import ( ) // Cancel the request -func cancelRequest(tr *http.Transport, req *http.Request) { - tr.CancelRequest(req) +func cancelRequest(transport http.RoundTripper, req *http.Request) { + if tr, ok := transport.(interface { + CancelRequest(*http.Request) + }); ok { + tr.CancelRequest(req) + } } // Reset a timer diff --git a/swift.go b/swift.go index 54829617d..9e8f70227 100644 --- a/swift.go +++ b/swift.go @@ -48,25 +48,47 @@ const ( // Rackspace v2 https://identity.api.rackspacecloud.com/v2.0 // Memset Memstore UK https://auth.storage.memset.com/v1.0 // Memstore v2 https://auth.storage.memset.com/v2.0 +// +// When using Google Appengine you must provide the Connection with an appengine-specific Transport: +// +// import ( +// "appengine/urlfetch" +// "fmt" +// "github.com/ncw/swift" +// ) +// +// func handler(w http.ResponseWriter, r *http.Request) { +// ctx := appengine.NewContext(r) +// tr := urlfetch.Transport{Context: ctx} +// c := swift.Connection{ +// UserName: "user", +// ApiKey: "key", +// AuthUrl: "auth_url", +// Transport: tr, +// } +// _ := c.Authenticate() +// containers, _ := c.ContainerNames(nil) +// fmt.Fprintf(w, "containers: %q", containers) +// } type Connection struct { // Parameters - fill these in before calling Authenticate // They are all optional except UserName, ApiKey and AuthUrl - UserName string // UserName for api - ApiKey string // Key for api access - AuthUrl string // Auth URL - Retries int // Retries on error (default is 3) - UserAgent string // Http User agent (default goswift/1.0) - ConnectTimeout time.Duration // Connect channel timeout (default 10s) - Timeout time.Duration // Data channel timeout (default 60s) - Region string // Region to use eg "LON", "ORD" - default is use first region (V2 auth only) - AuthVersion int // Set to 1 or 2 or leave at 0 for autodetect - Internal bool // Set this to true to use the the internal / service network - Tenant string // Name of the tenant (v2 auth only) - TenantId string // Id of the tenant (v2 auth only) + UserName string // UserName for api + ApiKey string // Key for api access + AuthUrl string // Auth URL + Retries int // Retries on error (default is 3) + UserAgent string // Http User agent (default goswift/1.0) + ConnectTimeout time.Duration // Connect channel timeout (default 10s) + Timeout time.Duration // Data channel timeout (default 60s) + Region string // Region to use eg "LON", "ORD" - default is use first region (V2 auth only) + AuthVersion int // Set to 1 or 2 or leave at 0 for autodetect + Internal bool // Set this to true to use the the internal / service network + Tenant string // Name of the tenant (v2 auth only) + TenantId string // Id of the tenant (v2 auth only) + Transport http.RoundTripper // Optional specialised http.Transport (eg. for Google Appengine) // These are filled in after Authenticate is called as are the defaults for above storageUrl string authToken string - tr *http.Transport client *http.Client Auth Authenticator // the current authenticator } @@ -191,7 +213,7 @@ func (c *Connection) doTimeoutRequest(timer *time.Timer, req *http.Request) (*ht return r.resp, r.err case <-timer.C: // Kill the connection on timeout so we don't leak sockets or goroutines - cancelRequest(c.tr, req) + cancelRequest(c.Transport, req) return nil, TimeoutError } panic("unreachable") // For Go 1.0 @@ -212,8 +234,8 @@ func (c *Connection) Authenticate() (err error) { if c.Timeout == 0 { c.Timeout = 60 * time.Second } - if c.tr == nil { - c.tr = &http.Transport{ + if c.Transport == nil { + c.Transport = &http.Transport{ // TLSClientConfig: &tls.Config{RootCAs: pool}, // DisableCompression: true, MaxIdleConnsPerHost: 2048, @@ -222,12 +244,12 @@ func (c *Connection) Authenticate() (err error) { if c.client == nil { c.client = &http.Client{ // CheckRedirect: redirectPolicyFunc, - Transport: c.tr, + Transport: c.Transport, } } // Flush the keepalives connection - if we are // re-authenticating then stuff has gone wrong - c.tr.CloseIdleConnections() + flushKeepaliveConnections(c.Transport) c.Auth, err = newAuth(c.AuthUrl, c.AuthVersion) if err != nil { return err @@ -245,7 +267,7 @@ func (c *Connection) Authenticate() (err error) { checkClose(resp.Body, &err) // Flush the auth connection - we don't want to keep // it open if keepalives were enabled - c.tr.CloseIdleConnections() + flushKeepaliveConnections(c.Transport) }() if err = c.parseHeaders(resp, authErrorMap); err != nil { return @@ -262,6 +284,15 @@ func (c *Connection) Authenticate() (err error) { return nil } +// flushKeepaliveConnections is called to flush pending requests after an error. +func flushKeepaliveConnections(transport http.RoundTripper) { + if tr, ok := transport.(interface { + CloseIdleConnections() + }); ok { + tr.CloseIdleConnections() + } +} + // UnAuthenticate removes the authentication from the Connection. func (c *Connection) UnAuthenticate() { c.storageUrl = "" @@ -374,7 +405,7 @@ func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response, } else { // Cancel the request on timeout cancel := func() { - cancelRequest(c.tr, req) + cancelRequest(c.Transport, req) } // Wrap resp.Body to make it obey an idle timeout resp.Body = newTimeoutReader(resp.Body, c.Timeout, cancel) diff --git a/swift_test.go b/swift_test.go index 6ae6a7128..fc733ff84 100644 --- a/swift_test.go +++ b/swift_test.go @@ -17,6 +17,7 @@ import ( "fmt" "github.com/ncw/swift" "io" + "net/http" "os" "testing" "time" @@ -40,7 +41,34 @@ const ( CONTENT_MD5 = "827ccb0eea8a706c4c34a16891f84e7b" ) -// Test functions are run in order - this one must be first! +type someTransport struct{ http.Transport } + +func TestTransport(t *testing.T) { + UserName := os.Getenv("SWIFT_API_USER") + ApiKey := os.Getenv("SWIFT_API_KEY") + AuthUrl := os.Getenv("SWIFT_AUTH_URL") + if UserName == "" || ApiKey == "" || AuthUrl == "" { + t.Fatal("SWIFT_API_USER, SWIFT_API_KEY and SWIFT_AUTH_URL not all set") + } + tr := &someTransport{Transport: http.Transport{MaxIdleConnsPerHost: 2048}} + ct := swift.Connection{ + UserName: UserName, + ApiKey: ApiKey, + AuthUrl: AuthUrl, + Tenant: os.Getenv("SWIFT_TENANT"), + TenantId: os.Getenv("SWIFT_TENANT_ID"), + Transport: tr, + } + err := ct.Authenticate() + if err != nil { + t.Fatal("Auth failed", err) + } + if !ct.Authenticated() { + t.Fatal("Not authenticated") + } +} + +// The following Test functions are run in order - this one must come before the others! func TestAuthenticate(t *testing.T) { UserName := os.Getenv("SWIFT_API_USER") ApiKey := os.Getenv("SWIFT_API_KEY")