From 61c6f206b073159c5463a083e0d04e5934351dfe Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 26 Aug 2014 16:21:04 -0700 Subject: [PATCH] registry: getting Endpoint ironned out Signed-off-by: Vincent Batts --- graph/pull.go | 4 +- graph/push.go | 4 +- registry/endpoint.go | 129 ++++++++++++++++++++++++++++++++++++++ registry/registry.go | 78 ----------------------- registry/registry_test.go | 14 ++++- registry/service.go | 11 ++-- registry/session.go | 30 ++++----- registry/types.go | 18 ++++++ 8 files changed, 181 insertions(+), 107 deletions(-) create mode 100644 registry/endpoint.go diff --git a/graph/pull.go b/graph/pull.go index 0ac99d7caad13..a2b27d64315ec 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -52,7 +52,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Error(err) } - endpoint, err := registry.ExpandAndVerifyRegistryUrl(hostname) + endpoint, err := registry.NewEndpoint(hostname) if err != nil { return job.Error(err) } @@ -62,7 +62,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Error(err) } - if endpoint == registry.IndexServerAddress() { + if endpoint.String() == registry.IndexServerAddress() { // If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar" localName = remoteName diff --git a/graph/push.go b/graph/push.go index 47cd4342c5fca..65d5e16b564fe 100644 --- a/graph/push.go +++ b/graph/push.go @@ -214,7 +214,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err) } - endpoint, err := registry.ExpandAndVerifyRegistryUrl(hostname) + endpoint, err := registry.NewEndpoint(hostname) if err != nil { return job.Error(err) } @@ -243,7 +243,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { var token []string job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName)) - if _, err := s.pushImage(r, job.Stdout, remoteName, img.ID, endpoint, token, sf); err != nil { + if _, err := s.pushImage(r, job.Stdout, remoteName, img.ID, endpoint.String(), token, sf); err != nil { return job.Error(err) } return engine.StatusOK diff --git a/registry/endpoint.go b/registry/endpoint.go new file mode 100644 index 0000000000000..12df9e0c91988 --- /dev/null +++ b/registry/endpoint.go @@ -0,0 +1,129 @@ +package registry + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/pkg/log" +) + +// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. +func scanForApiVersion(hostname string) (string, APIVersion) { + var ( + chunks []string + apiVersionStr string + ) + if strings.HasSuffix(hostname, "/") { + chunks = strings.Split(hostname[:len(hostname)-1], "/") + apiVersionStr = chunks[len(chunks)-1] + } else { + chunks = strings.Split(hostname, "/") + apiVersionStr = chunks[len(chunks)-1] + } + for k, v := range apiVersions { + if apiVersionStr == v { + hostname = strings.Join(chunks[:len(chunks)-1], "/") + return hostname, k + } + } + return hostname, DefaultAPIVersion +} + +func NewEndpoint(hostname string) (*Endpoint, error) { + var ( + endpoint Endpoint + trimmedHostname string + err error + ) + if !strings.HasPrefix(hostname, "http") { + hostname = "https://" + hostname + } + trimmedHostname, endpoint.Version = scanForApiVersion(hostname) + endpoint.URL, err = url.Parse(trimmedHostname) + if err != nil { + return nil, err + } + + endpoint.URL.Scheme = "https" + if _, err := endpoint.Ping(); err != nil { + log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + // TODO: Check if http fallback is enabled + endpoint.URL.Scheme = "http" + if _, err = endpoint.Ping(); err != nil { + return nil, errors.New("Invalid Registry endpoint: " + err.Error()) + } + } + + return &endpoint, nil +} + +type Endpoint struct { + URL *url.URL + Version APIVersion +} + +// Get the formated URL for the root of this registry Endpoint +func (e Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +} + +func (e Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +} + +func (e Endpoint) Ping() (RegistryInfo, error) { + if e.String() == IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return RegistryInfo{Standalone: false}, nil + } + + req, err := http.NewRequest("GET", e.String()+"_ping", nil) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + + defer resp.Body.Close() + + jsonString, err := ioutil.ReadAll(resp.Body) + if err != nil { + return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + } + + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry. default to true + info := RegistryInfo{ + Standalone: true, + } + if err := json.Unmarshal(jsonString, &info); err != nil { + log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + // don't stop here. Just assume sane defaults + } + if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { + log.Debugf("Registry version header: '%s'", hdr) + info.Version = hdr + } + log.Debugf("RegistryInfo.Version: %q", info.Version) + + standalone := resp.Header.Get("X-Docker-Registry-Standalone") + log.Debugf("Registry standalone header: '%s'", standalone) + // Accepted values are "true" (case-insensitive) and "1". + if strings.EqualFold(standalone, "true") || standalone == "1" { + info.Standalone = true + } else if len(standalone) > 0 { + // there is a header set, and it is not "true" or "1", so assume fails + info.Standalone = false + } + log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) + return info, nil +} diff --git a/registry/registry.go b/registry/registry.go index a2e1fbdd1ddea..203dfa6fc1d7b 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -3,7 +3,6 @@ package registry import ( "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -15,7 +14,6 @@ import ( "strings" "time" - "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" ) @@ -152,55 +150,6 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt return nil, nil, nil } -func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { - if endpoint == IndexServerAddress() { - // Skip the check, we now this one is valid - // (and we never want to fallback to http in case of error) - return RegistryInfo{Standalone: false}, nil - } - - req, err := http.NewRequest("GET", endpoint+"_ping", nil) - if err != nil { - return RegistryInfo{Standalone: false}, err - } - - resp, _, err := doRequest(req, nil, ConnectTimeout) - if err != nil { - return RegistryInfo{Standalone: false}, err - } - - defer resp.Body.Close() - - jsonString, err := ioutil.ReadAll(resp.Body) - if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) - } - - // If the header is absent, we assume true for compatibility with earlier - // versions of the registry. default to true - info := RegistryInfo{ - Standalone: true, - } - if err := json.Unmarshal(jsonString, &info); err != nil { - log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) - // don't stop here. Just assume sane defaults - } - if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - log.Debugf("Registry version header: '%s'", hdr) - info.Version = hdr - } - log.Debugf("RegistryInfo.Version: %q", info.Version) - - standalone := resp.Header.Get("X-Docker-Registry-Standalone") - log.Debugf("Registry standalone header: '%s'", standalone) - if !strings.EqualFold(standalone, "true") && standalone != "1" && len(standalone) > 0 { - // there is a header set, and it is not "true" or "1", so assume fails - info.Standalone = false - } - log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) - return info, nil -} - func validateRepositoryName(repositoryName string) error { var ( namespace string @@ -252,33 +201,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } -// this method expands the registry name as used in the prefix of a repo -// to a full url. if it already is a url, there will be no change. -// The registry is pinged to test if it http or https -func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { - if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { - // if there is no slash after https:// (8 characters) then we have no path in the url - if strings.LastIndex(hostname, "/") < 9 { - // there is no path given. Expand with default path - hostname = hostname + "/v1/" - } - if _, err := pingRegistryEndpoint(hostname); err != nil { - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } - return hostname, nil - } - endpoint := fmt.Sprintf("https://%s/v1/", hostname) - if _, err := pingRegistryEndpoint(endpoint); err != nil { - log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - endpoint = fmt.Sprintf("http://%s/v1/", hostname) - if _, err = pingRegistryEndpoint(endpoint); err != nil { - //TODO: triggering highland build can be done there without "failing" - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } - } - return endpoint, nil -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/registry/registry_test.go b/registry/registry_test.go index 8a95221dc7b51..ab4178126af71 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -18,7 +18,11 @@ var ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) + endpoint, err := NewEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } + r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), endpoint, true) if err != nil { t.Fatal(err) } @@ -26,7 +30,11 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - regInfo, err := pingRegistryEndpoint(makeURL("/v1/")) + ep, err := NewEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } + regInfo, err := ep.Ping() if err != nil { t.Fatal(err) } @@ -197,7 +205,7 @@ func TestPushImageJSONIndex(t *testing.T) { if repoData == nil { t.Fatal("Expected RepositoryData object") } - repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint}) + repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint.String()}) if err != nil { t.Fatal(err) } diff --git a/registry/service.go b/registry/service.go index 0e6f1bda93365..f7b353000e94d 100644 --- a/registry/service.go +++ b/registry/service.go @@ -40,11 +40,14 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) // TODO: this is only done here because auth and registry need to be merged into one pkg if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - addr, err = ExpandAndVerifyRegistryUrl(addr) + endpoint, err := NewEndpoint(addr) if err != nil { return job.Error(err) } - authConfig.ServerAddress = addr + if _, err := endpoint.Ping(); err != nil { + return job.Error(err) + } + authConfig.ServerAddress = endpoint.String() } status, err := Login(authConfig, HTTPRequestFactory(nil)) if err != nil { @@ -86,11 +89,11 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - hostname, err = ExpandAndVerifyRegistryUrl(hostname) + endpoint, err := NewEndpoint(hostname) if err != nil { return job.Error(err) } - r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) + r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), endpoint, true) if err != nil { return job.Error(err) } diff --git a/registry/session.go b/registry/session.go index 58263ef6e30bd..4862630830b1a 100644 --- a/registry/session.go +++ b/registry/session.go @@ -25,15 +25,15 @@ import ( type Session struct { authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory - indexEndpoint string + indexEndpoint *Endpoint jar *cookiejar.Jar timeout TimeoutType } -func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Session, err error) { +func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) { r = &Session{ authConfig: authConfig, - indexEndpoint: indexEndpoint, + indexEndpoint: endpoint, } if timeout { @@ -47,13 +47,13 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, index // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. - if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - info, err := pingRegistryEndpoint(indexEndpoint) + if r.indexEndpoint.String() != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" { + info, err := r.indexEndpoint.Ping() if err != nil { return nil, err } if info.Standalone { - log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) + log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", r.indexEndpoint.String()) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } @@ -261,8 +261,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { } func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { - indexEp := r.indexEndpoint - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.String(), remote) log.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -296,17 +295,13 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } } else { // Assume the endpoint is on the same host - u, err := url.Parse(indexEp) - if err != nil { - return nil, err - } - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", r.indexEndpoint.URL.Scheme, req.URL.Host)) } checksumsJSON, err := ioutil.ReadAll(res.Body) @@ -474,7 +469,6 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} - indexEp := r.indexEndpoint if validate { for _, elem := range imgList { @@ -494,7 +488,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.String(), remote, suffix) log.Debugf("[registry] PUT %s", u) log.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) @@ -552,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } @@ -578,7 +572,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err diff --git a/registry/types.go b/registry/types.go index 70d55e42fdd26..3db236da32c01 100644 --- a/registry/types.go +++ b/registry/types.go @@ -31,3 +31,21 @@ type RegistryInfo struct { Version string `json:"version"` Standalone bool `json:"standalone"` } + +type APIVersion int + +func (av APIVersion) String() string { + return apiVersions[av] +} + +var DefaultAPIVersion APIVersion = APIVersion1 +var apiVersions = map[APIVersion]string{ + 1: "v1", + 2: "v2", +} + +const ( + _ = iota + APIVersion1 = iota + APIVersion2 +)