From 0c4b40abf12e352e3c0f92eddce54709f962e555 Mon Sep 17 00:00:00 2001 From: "jesus m. rodriguez" Date: Mon, 18 Mar 2019 11:57:00 -0400 Subject: [PATCH] Bug 1678158 - backport oauth2 support to rhcc * fix debug statement * add version log string * copy the skip_verify_tls setting to the reg config * pass client around so we can skip tls on image get --- pkg/app/app.go | 3 + pkg/registries/adapters/adapter.go | 22 +- pkg/registries/adapters/oauth/client.go | 268 +++++++++++++++++++ pkg/registries/adapters/oauth/client_test.go | 120 +++++++++ pkg/registries/adapters/rhcc_adapter.go | 33 ++- pkg/registries/adapters/rhcc_adapter_test.go | 2 +- pkg/registries/registry.go | 55 ++-- 7 files changed, 460 insertions(+), 43 deletions(-) create mode 100644 pkg/registries/adapters/oauth/client.go create mode 100644 pkg/registries/adapters/oauth/client_test.go diff --git a/pkg/app/app.go b/pkg/app/app.go index ed6fdd414f..48416c6b38 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -176,6 +176,9 @@ func CreateApp() App { os.Exit(1) } + // Show version in logs for better debugging + log.Infof("Ansible Service Broker Version: %v", version.Version) + // Initializing clients as soon as we have deps ready. err = initClients(app.config) if err != nil { diff --git a/pkg/registries/adapters/adapter.go b/pkg/registries/adapters/adapter.go index 9c40b681c8..47157a3c36 100644 --- a/pkg/registries/adapters/adapter.go +++ b/pkg/registries/adapters/adapter.go @@ -48,22 +48,28 @@ const BundleSpecLabel = "com.redhat.apb.spec" // Configuration - Adapter configuration. Contains the info that the adapter // would need to complete its request to the images. type Configuration struct { - URL *url.URL - User string - Pass string - Org string - Images []string - Namespaces []string - Tag string + URL *url.URL + User string + Pass string + Org string + Images []string + Namespaces []string + Tag string + SkipVerifyTLS bool } // Retrieve the spec from a registry manifest request func imageToSpec(log *logging.Logger, req *http.Request, image string) (*apb.Spec, error) { + return imageToSpecWithClient(http.DefaultClient, log, req, image) +} + +// Retrieve the spec from a registry manifest request +func imageToSpecWithClient(client *http.Client, log *logging.Logger, req *http.Request, image string) (*apb.Spec, error) { log.Debug("Registry::imageToSpec") spec := &apb.Spec{} req.Header.Add("Accept", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { return nil, err } diff --git a/pkg/registries/adapters/oauth/client.go b/pkg/registries/adapters/oauth/client.go new file mode 100644 index 0000000000..8b69e48b98 --- /dev/null +++ b/pkg/registries/adapters/oauth/client.go @@ -0,0 +1,268 @@ +// +// Copyright (c) 2018 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package oauth + +import ( + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "regexp" + "sync" + "time" + + logging "github.com/op/go-logging" +) + +// oauth2Response - holds the response data from an oauth2 token request +type oauth2Response struct { + Token string `json:"access_token"` +} + +// dockerResponse - holds the response data from a docker token request +type dockerResponse struct { + Token string `json:"token"` +} + +// NewClient - creates and returns a *Client ready to use. If skipVerify is +// true, it will skip verification of the remote TLS certificate. +func NewClient(user, pass string, skipVerify bool, url *url.URL, log *logging.Logger) *Client { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: skipVerify}, + Proxy: http.ProxyFromEnvironment, + } + if skipVerify == true { + log.Warning("skipping verification of registry TLS certificate per adapter configuration") + } + + return &Client{ + user: user, + pass: pass, + url: url, + mutex: &sync.Mutex{}, + client: &http.Client{Timeout: time.Second * 60, Transport: transport}, + log: log, + } +} + +// Client - may be used as an HTTP client that automatically authenticates using oauth. +type Client struct { + user string + pass string + token string + mutex *sync.Mutex + client *http.Client + url *url.URL + log *logging.Logger +} + +// NewRequest - creates and returns a *http.Request assuming the GET method. +// The base URL configured on the Client gets used with its Path component +// replaced by the path argument. If a token is available, it is added to the +// request automatically. "Accept: application/json" is added to all requests. +// The caller should customize the request as necessary before using it. +func (c *Client) NewRequest(path string) (*http.Request, error) { + req, err := http.NewRequest("GET", c.url.String(), nil) + if err != nil { + return nil, err + } + req.URL.Path = path + if c.token != "" { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", c.token)) + } + req.Header.Add("Accept", "application/json") + return req, nil +} + +// Do - passes through to the underlying http.Client instance +func (c *Client) Do(req *http.Request) (*http.Response, error) { + return c.client.Do(req) +} + +// GetClient - returns the http.Client +func (c *Client) GetClient() *http.Client { + return c.client +} + +// Getv2 - makes a GET request to the registry's /v2/ endpoint. If a 401 +// Unauthorized response is received, this method attempts to obtain an oauth +// token and tries again with the new token. If a username and password are +// available, they are used with Basic Auth in the request to the token +// service. This method is goroutine-safe. +func (c *Client) Getv2() error { + // lock to prevent multiple goroutines from retrieving, and especially + // writing, a new token at the same time + c.mutex.Lock() + defer c.mutex.Unlock() + req, err := c.NewRequest("/v2/") + if err != nil { + return err + } + resp, err := c.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusUnauthorized: + h := resp.Header.Get("www-authenticate") + c.getToken(h) + + // try the new token + tokenReq, err := c.NewRequest("/v2/") + if err != nil { + return err + } + tokenResp, err := c.Do(tokenReq) + if err != nil { + return err + } + defer tokenResp.Body.Close() + + if tokenResp.StatusCode != http.StatusOK { + msg := fmt.Sprintf("Token not accepted by /v2/ - %s", tokenResp.Status) + c.log.Warning(msg) + return errors.New(msg) + } + c.log.Debug("GET /v2/ successful with new token") + + case http.StatusOK: + if c.token == "" { + c.log.Debug("GET /v2/ successful without token") + } else { + c.log.Debug("GET /v2/ successful with existing token") + } + + default: + msg := fmt.Sprintf("Bad response from /v2/ - %s", resp.Status) + c.log.Warning(msg) + return errors.New(msg) + } + return nil +} + +// getToken - parses a www-authenticate header and uses the information to +// retrieve an oauth token. The token is stored on the Client and automatically +// added to future requests. +func (c *Client) getToken(wwwauth string) error { + // compute the URL + u, err := parseAuthHeader(wwwauth) + if err != nil { + return err + } + + // form the request + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + c.log.Errorf("could not form request: %s", err.Error()) + return err + } + if c.pass != "" { + c.log.Debug("adding basic auth to token request") + req.SetBasicAuth(c.user, c.pass) + } + + // make the request + resp, err := c.client.Do(req) + if err != nil { + msg := fmt.Sprintf("error obtaining token: %s", err.Error()) + c.log.Warning(msg) + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + msg := fmt.Sprintf("token service responded: %s", resp.Status) + c.log.Warning(msg) + return errors.New(msg) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + c.log.Warningf("failed to read token body: %s", err.Error()) + return err + } + + c.token, err = parseAuthToken(body) + if err != nil { + return err + } + + c.log.Debugf("new token: %s", c.token) + return nil +} + +// parseAuthHeader - parses the text from a www-authenticate header and uses it +// to construct a url.URL that can be used to retrieve a token. +func parseAuthHeader(value string) (*url.URL, error) { + rrealm, err := regexp.Compile("realm=\"([^\"]+)\"") + if err != nil { + return nil, err + } + rservice, err := regexp.Compile("service=\"([^\"]+)\"") + if err != nil { + return nil, err + } + + rmatch := rrealm.FindStringSubmatch(value) + if len(rmatch) != 2 { + msg := fmt.Sprintf("Could not parse www-authenticate header: %s", value) + return nil, errors.New(msg) + } + realm := rmatch[1] + + u, err := url.Parse(realm) + if err != nil { + msg := fmt.Sprintf("realm is not a valid URL: %s", realm) + return nil, errors.New(msg) + } + + smatch := rservice.FindStringSubmatch(value) + if len(smatch) == 2 { + service := smatch[1] + q := u.Query() + q.Set("service", service) + u.RawQuery = q.Encode() + } + + return u, nil +} + +// parseAuthToken - parses a token from a http response body in json format. +// The token can be located in the fields "access_token" or "token" of the json +// response. This method returns the value of one of those fields (favouring +// "access_token" if both are set) or an empty string if non of the fields is set. +func parseAuthToken(body []byte) (string, error) { + oauth2Resp := oauth2Response{} + err := json.Unmarshal(body, &oauth2Resp) + if err != nil { + return "", err + } + + if oauth2Resp.Token != "" { + return oauth2Resp.Token, nil + } + + dockerResp := dockerResponse{} + err = json.Unmarshal(body, &dockerResp) + if err != nil { + return "", err + } + return dockerResp.Token, nil +} diff --git a/pkg/registries/adapters/oauth/client_test.go b/pkg/registries/adapters/oauth/client_test.go new file mode 100644 index 0000000000..c5e83eb42e --- /dev/null +++ b/pkg/registries/adapters/oauth/client_test.go @@ -0,0 +1,120 @@ +// +// Copyright (c) 2018 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package oauth + +import ( + "net/url" + "strings" + "testing" + + logging "github.com/op/go-logging" +) + +var headerCases = map[string]string{ + "Bearer realm=\"http://foo/a/b/c\",service=\"bar\"": "http://foo/a/b/c?service=bar", + "Bearer service=\"bar\",realm=\"http://foo/a/b/c\"": "http://foo/a/b/c?service=bar", + "Bearer realm=\"http://foo/a/b/c/\",service=\"bar\"": "http://foo/a/b/c/?service=bar", + "Bearer realm=\"https://foo\",service=\"bar\"": "https://foo?service=bar", + "Bearer realm=\"http://foo/a/b/c\"": "http://foo/a/b/c", +} + +var headerErrorCases = map[string]string{ + "Bearer service=\"bar\"": "Could not parse www-authenticate header:", + "Bearer realm=\"\"": "", +} + +var tokenCases = map[string]string{ + "{\"access_token\": \"abc123\"}": "abc123", + "{\"token\": \"abc123\"}": "abc123", + "{\"access_token\": \"abc123\", \"token\": \"def456\"}": "abc123", + "{}": "", +} + +var tokenErrorCases = map[string]string{ + "{\"token\": {}": "unexpected end of JSON input", + "{\"access_token\": {}": "unexpected end of JSON input", + "{\"token\": null": "unexpected end of JSON input", + "{\"access_token\": null": "unexpected end of JSON input", +} + +func TestParseAuthHeader(t *testing.T) { + for in, out := range headerCases { + result, err := parseAuthHeader(in) + if err != nil { + t.Error(err.Error()) + } + if result.String() != out { + t.Errorf("Expected %s, got %s", out, result.String()) + } + } +} + +func TestParseAuthHeaderErrors(t *testing.T) { + for in, out := range headerErrorCases { + _, err := parseAuthHeader(in) + if err == nil { + t.Errorf("Expected an error parsing %s", in) + } else if strings.HasPrefix(err.Error(), out) == false { + t.Errorf("Expected prefix %s, got %s", out, err.Error()) + } + } +} + +func TestParseAuthToken(t *testing.T) { + for in, out := range tokenCases { + result, err := parseAuthToken([]byte(in)) + if err != nil { + t.Error(err.Error()) + } + if result != out { + t.Errorf("Expected %s, got %s", out, result) + } + } +} + +func TestParseAuthTokenErrors(t *testing.T) { + for in, out := range tokenErrorCases { + _, err := parseAuthToken([]byte(in)) + if err == nil { + t.Errorf("Expected an error parsing %s", in) + } else if strings.HasPrefix(err.Error(), out) == false { + t.Errorf("Expected prefix %s, got %s", out, err.Error()) + } + } +} + +func TestNewRequest(t *testing.T) { + log := &logging.Logger{} + u, _ := url.Parse("http://automationbroker.io") + c := NewClient("foo", "bar", false, u, log) + c.token = "letmein" + req, err := c.NewRequest("/v2/") + if err != nil { + t.Error(err.Error()) + return + } + accepth := req.Header.Get("Accept") + if accepth != "application/json" { + t.Errorf("incorrect or missing accept header: %s", accepth) + return + } + authh := req.Header.Get("Authorization") + if authh != "Bearer letmein" { + t.Errorf("incorrect or missing authorization header: %s", authh) + return + } +} diff --git a/pkg/registries/adapters/rhcc_adapter.go b/pkg/registries/adapters/rhcc_adapter.go index 31287a91a0..97000ffb2d 100644 --- a/pkg/registries/adapters/rhcc_adapter.go +++ b/pkg/registries/adapters/rhcc_adapter.go @@ -21,16 +21,26 @@ import ( "errors" "fmt" "io/ioutil" - "net/http" logging "github.com/op/go-logging" "github.com/openshift/ansible-service-broker/pkg/apb" + "github.com/openshift/ansible-service-broker/pkg/registries/adapters/oauth" ) +// NewRHCCAdapter - creates and returns a *RHCCAdapter ready to use. +func NewRHCCAdapter(config Configuration, log *logging.Logger) *RHCCAdapter { + return &RHCCAdapter{ + Config: config, + Log: log, + client: oauth.NewClient(config.User, config.Pass, config.SkipVerifyTLS, config.URL, log), + } +} + // RHCCAdapter - Red Hat Container Catalog Registry type RHCCAdapter struct { Config Configuration Log *logging.Logger + client *oauth.Client } // RHCCImage - RHCC Registry Image that is returned from the RHCC Catalog api. @@ -60,6 +70,7 @@ func (r RHCCAdapter) RegistryName() string { // GetImageNames - retrieve the images from the registry func (r RHCCAdapter) GetImageNames() ([]string, error) { + r.client.Getv2() imageList, err := r.loadImages("\"*-apb\"") if err != nil { return nil, err @@ -89,16 +100,18 @@ func (r RHCCAdapter) FetchSpecs(imageNames []string) ([]*apb.Spec, error) { } // LoadImages - Get all the images for a particular query -func (r RHCCAdapter) loadImages(Query string) (RHCCImageResponse, error) { +func (r RHCCAdapter) loadImages(query string) (RHCCImageResponse, error) { r.Log.Debug("RHCCRegistry::LoadImages") - r.Log.Debug("Using " + r.Config.URL.String() + " to source APB images using query:" + Query) - req, err := http.NewRequest("GET", - fmt.Sprintf("%v/v1/search?q=%v", r.Config.URL.String(), Query), nil) + req, err := r.client.NewRequest("/v1/search") if err != nil { return RHCCImageResponse{}, err } + q := req.URL.Query() + q.Set("q", query) + req.URL.RawQuery = q.Encode() + r.Log.Debugf("Using %s to source APB images", req.URL.String()) - resp, err := http.DefaultClient.Do(req) + resp, err := r.client.Do(req) if err != nil { return RHCCImageResponse{}, err } @@ -124,11 +137,13 @@ func (r RHCCAdapter) loadSpec(imageName string) (*apb.Spec, error) { if r.Config.Tag == "" { r.Config.Tag = "latest" } - req, err := http.NewRequest("GET", - fmt.Sprintf("%v/v2/%v/manifests/%v", r.Config.URL.String(), imageName, r.Config.Tag), nil) + req, err := r.client.NewRequest(fmt.Sprintf("/v2/%v/manifests/%v", imageName, r.Config.Tag)) if err != nil { return nil, err } - return imageToSpec(r.Log, req, fmt.Sprintf("%s/%s:%s", r.RegistryName(), imageName, r.Config.Tag)) + // NOTE: 3.11 patch has a call to + req.Header.Add("Accept", "application/json") + + return imageToSpecWithClient(r.client.GetClient(), r.Log, req, fmt.Sprintf("%s/%s:%s", r.RegistryName(), imageName, r.Config.Tag)) } diff --git a/pkg/registries/adapters/rhcc_adapter_test.go b/pkg/registries/adapters/rhcc_adapter_test.go index a6dcc4ab10..ccb157dc86 100644 --- a/pkg/registries/adapters/rhcc_adapter_test.go +++ b/pkg/registries/adapters/rhcc_adapter_test.go @@ -141,7 +141,7 @@ func TestGetImages(t *testing.T) { t.Fatal("ERROR: ", err) } config := Configuration{URL: u} - adapter := RHCCAdapter{Config: config, Log: log} + adapter := NewRHCCAdapter(config, log) imageNames, err := adapter.GetImageNames() ft.AssertEqual(t, len(imageNames), 3) ft.AssertNotNil(t, imageNames) diff --git a/pkg/registries/registry.go b/pkg/registries/registry.go index cfbcb449a2..3c39a1597b 100644 --- a/pkg/registries/registry.go +++ b/pkg/registries/registry.go @@ -61,9 +61,10 @@ type Config struct { Namespaces []string // Fail will tell the registry that it is ok to fail the bootstrap if // just this registry has failed. - Fail bool `yaml:"fail_on_error"` - WhiteList []string `yaml:"white_list"` - BlackList []string `yaml:"black_list"` + Fail bool `yaml:"fail_on_error"` + WhiteList []string `yaml:"white_list"` + BlackList []string `yaml:"black_list"` + SkipVerifyTLS bool `yaml:"skip_verify_tls"` } // Validate - makes sure the registry config is valid. @@ -186,20 +187,21 @@ func (r Registry) RegistryName() string { func NewRegistry(con *config.Config, asbNamespace string) (Registry, error) { var adapter adapters.Adapter configuration := Config{ - URL: con.GetString("url"), - User: con.GetString("user"), - Pass: con.GetString("pass"), - Org: con.GetString("org"), - Tag: con.GetString("tag"), - Type: con.GetString("type"), - Name: con.GetString("name"), - Images: con.GetSliceOfStrings("images"), - Namespaces: con.GetSliceOfStrings("namespaces"), - Fail: con.GetBool("fail_on_error"), - WhiteList: con.GetSliceOfStrings("white_list"), - BlackList: con.GetSliceOfStrings("black_list"), - AuthType: con.GetString("auth_type"), - AuthName: con.GetString("auth_name"), + URL: con.GetString("url"), + User: con.GetString("user"), + Pass: con.GetString("pass"), + Org: con.GetString("org"), + Tag: con.GetString("tag"), + Type: con.GetString("type"), + Name: con.GetString("name"), + Images: con.GetSliceOfStrings("images"), + Namespaces: con.GetSliceOfStrings("namespaces"), + Fail: con.GetBool("fail_on_error"), + WhiteList: con.GetSliceOfStrings("white_list"), + BlackList: con.GetSliceOfStrings("black_list"), + AuthType: con.GetString("auth_type"), + AuthName: con.GetString("auth_name"), + SkipVerifyTLS: con.GetBool("skip_verify_tls"), } if !configuration.Validate() { return Registry{}, errors.New("unable to validate registry name") @@ -226,17 +228,20 @@ func NewRegistry(con *config.Config, asbNamespace string) (Registry, error) { if u.Scheme == "" { u.Scheme = "http" } - c := adapters.Configuration{URL: u, - User: configuration.User, - Pass: configuration.Pass, - Org: configuration.Org, - Images: configuration.Images, - Namespaces: configuration.Namespaces, - Tag: configuration.Tag} + c := adapters.Configuration{ + URL: u, + User: configuration.User, + Pass: configuration.Pass, + Org: configuration.Org, + Images: configuration.Images, + Namespaces: configuration.Namespaces, + Tag: configuration.Tag, + SkipVerifyTLS: configuration.SkipVerifyTLS, + } switch strings.ToLower(configuration.Type) { case "rhcc": - adapter = &adapters.RHCCAdapter{Config: c, Log: log} + adapter = adapters.NewRHCCAdapter(c, log) case "dockerhub": adapter = &adapters.DockerHubAdapter{Config: c, Log: log} case "mock":