Skip to content

Commit

Permalink
New flags for setting registry request rate limits.
Browse files Browse the repository at this point in the history
  • Loading branch information
@yoshiotu
yoshiotu committed Aug 10, 2016
1 parent b32c404 commit 83374a9
Show file tree
Hide file tree
Showing 6 changed files with 337 additions and 177 deletions.
13 changes: 13 additions & 0 deletions cmd/collector/custom.go
View file
Expand Up @@ -72,6 +72,19 @@ func doFlags() {
collector.LocalHost = true
}
//nextMaxImages = *maxImages

if *maxRequests != 0 {
err := collector.AddRegistryRateLimiter(*maxRequests, *timePeriod)
if err != nil {
except.Fail(err, ": Error in setting registry rate limiter")
}
}
if *maxRequests2 != 0 {
err := collector.AddRegistryRateLimiter(*maxRequests2, *timePeriod2)
if err != nil {
except.Fail(err, ": Error in setting registry rate limiter")
}
}
}

func printExampleUsage() {
Expand Down
6 changes: 6 additions & 0 deletions cmd/collector/main.go
View file
Expand Up @@ -48,6 +48,12 @@ var (
dockerAddr = flag.String([]string{"-dockeraddr"}, "/var/run/docker.sock",
"Address of Docker remote API socket (filepath or IP:port)")

// Docker Registry rate limiting
maxRequests = flag.Int([]string{"-maxreq"}, 0, "max # of requests to registry in time period (0 for no limit)")
maxRequests2 = flag.Int([]string{"-maxreq2"}, 0, "max # of requests to registry in time period 2 (0 for no limit)")
timePeriod = flag.Duration([]string{"-timeper"}, 10*time.Minute, "registry request rate limiting time period")
timePeriod2 = flag.Duration([]string{"-timeper2"}, 24*time.Hour, "registry request rate limiting time period 2")

// positional arguments: a list of repos to process, all others are ignored.
)

Expand Down
1 change: 1 addition & 0 deletions docs/CollectorDetails.md
View file
Expand Up @@ -26,6 +26,7 @@ The collector has been designed so that it is modular and extensible with differ

The figure above shows the overall collector architecture. At the center is the collector core that takes in inputs from various plugins, and then launches/collects data for desired containers (as described in the previous section). Here are some of the plugins where we encourage users to contribute/submit pull requests:
* Registry: We currently support both private registry and DockerHub as the source of image location. But a given collector instance can only run on a single registry (one private registry or docker hub). However, you can run multiple instances of the collector pointing to different private registries and/or docker hub.
* Collector has command line options to limit the rate at which Collector issues requests to the registry. You can specify zero, one, or two rate limits. Each rate limit specifies the maximum number of requests allowed in a specified time period. For example, you could set a rate limit of 500 requests each 10 minutes (--maxreq=50 --timeper=10m), and add a second rate limit of 10000 requests per day (--maxreq2=10000 --timeper2=24h0m0s).
* Possible extensions: multiple registry support, images in the local filesystem (e.g., not uploaded to registry)
* User-specified scripts: We support multiple types of plugins to write scripts for data collection including Bash and Python. We provide statically linked versions of bash and python, and busybox commands by exploring volumes into the containers to be inspected. That way, we don’t rely on any pre-existing tools inside the container to run scripts. We’ve also provided two sample bash scripts: PkgExtract and PkgDeps that collect package information and dependencies between different packages.
* Possible extensions: Ruby, Go itself, etc.
Expand Down
180 changes: 3 additions & 177 deletions imagedata.go
View file
Expand Up @@ -3,18 +3,16 @@
package collector

import (
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"strconv"
"strings"

config "github.com/banyanops/collector/config"
except "github.com/banyanops/collector/except"
blog "github.com/ccpaging/log4go"
)

var ()

// ImageDataInfo describes a package included in the contents of an image.
type ImageDataInfo struct {
Image string //this has to be the first field (used in order by)
Expand All @@ -25,13 +23,9 @@ type ImageDataInfo struct {
Architecture string
}

// Registry V2 authorization server result
type authServerResult struct {
Token string `json:"token"`
}

// PullImage performs a docker pull on an image specified by repo/tag.
func PullImage(metadata *ImageMetadataInfo) (err error) {
RegistryLimiterWait()
tagspec := metadata.Repo + ":" + metadata.Tag
if RegistrySpec != config.DockerHub {
tagspec = RegistrySpec + "/" + tagspec
Expand Down Expand Up @@ -175,174 +169,6 @@ func RemoveDanglingImages() (e error) {
return
}

type HTTPStatusCodeError struct {
error
StatusCode int
}

func (s *HTTPStatusCodeError) Error() string {
return "HTTP Status Code " + strconv.Itoa(s.StatusCode)
}

// RegistryQueryV1 performs an HTTP GET operation from a V1 registry and returns the response.
func RegistryQueryV1(client *http.Client, URL string) (response []byte, e error) {
_, _, BasicAuth, XRegistryAuth = GetRegistryURL()
req, e := http.NewRequest("GET", URL, nil)
if e != nil {
return nil, e
}
if BasicAuth != "" {
req.Header.Set("Authorization", "Basic "+BasicAuth)
}
r, e := client.Do(req)
if e != nil {
return nil, e
}
defer r.Body.Close()
if r.StatusCode < 200 || r.StatusCode > 299 {
e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
return
}
response, e = ioutil.ReadAll(r.Body)
if e != nil {
return
}
return
}

// RegistryQueryV2 performs an HTTP GET operation from the registry and returns the response.
// If the initial response has status code 401 Unauthorized and includes WWW-Authenticate header,
// then we follow the directions in that header to get an access token, and finally
// re-issue the initial call to get the final response.
func RegistryQueryV2(client *http.Client, URL string) (response []byte, e error) {
_, _, BasicAuth, XRegistryAuth = GetRegistryURL()
req, e := http.NewRequest("GET", URL, nil)
if e != nil {
return nil, e
}
req.Header.Set("Authorization", "Basic "+BasicAuth)
r, e := client.Do(req)
if e != nil {
return nil, e
}
if r.StatusCode == 401 {
blog.Debug("Registry Query %s got 401", URL)
// get the WWW-Authenticate header
WWWAuth := r.Header.Get("WWW-Authenticate")
if WWWAuth == "" {
except.Error("Empty WWW-Authenticate", URL)
return
}
arr := strings.Fields(WWWAuth)
if len(arr) != 2 {
e = errors.New("Invalid WWW-Authenticate format for " + WWWAuth)
except.Error(e)
return
}
authType := arr[0]
blog.Debug("Authorization type: %s", authType)
fieldMap := make(map[string]string)
e = parseAuthenticateFields(arr[1], fieldMap)
if e != nil {
except.Error(e)
return
}
r.Body.Close()
// access the authentication server to get a token
token, err := queryAuthServerV2(client, fieldMap, BasicAuth)
if err != nil {
except.Error(err)
return nil, err
}
// re-issue the original request, this time using the token
req, e = http.NewRequest("GET", URL, nil)
if e != nil {
return nil, e
}
req.Header.Set("Authorization", authType+" "+token)
r, e = client.Do(req)
if e != nil {
return nil, e
}
}
defer r.Body.Close()
if r.StatusCode < 200 || r.StatusCode > 299 {
e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
return
}
response, e = ioutil.ReadAll(r.Body)
if e != nil {
return
}
blog.Debug("Registry query succeeded")
return
}

/* queryAuthServerV2 retrieves an authorization token from a V2 auth server */
func queryAuthServerV2(client *http.Client, fieldMap map[string]string, BasicAuth string) (token string, e error) {
authServer := fieldMap["realm"]
if authServer == "" {
e = errors.New("No registry token auth server specified")
return
}
blog.Debug("authServer=%s\n", authServer)
URL := authServer
first := true
for key, value := range fieldMap {
if key != "realm" {
if first {
URL = URL + "?"
first = false
} else {
URL = URL + "&"
}
URL = URL + key + "=" + value
}
}
blog.Debug("Auth server URL is %s", URL)

req, e := http.NewRequest("GET", URL, nil)
if e != nil {
return
}
req.Header.Set("Authorization", "Basic "+BasicAuth)
r, e := client.Do(req)
if e != nil {
return
}
defer r.Body.Close()
if r.StatusCode < 200 || r.StatusCode > 299 {
e = &HTTPStatusCodeError{StatusCode: r.StatusCode}
return
}
response, e := ioutil.ReadAll(r.Body)
if e != nil {
return
}
var parsedReply authServerResult
e = json.Unmarshal(response, &parsedReply)
if e != nil {
return
}
token = parsedReply.Token
return token, e
}

func parseAuthenticateFields(s string, fieldMap map[string]string) (e error) {
fields := strings.Split(s, ",")
for _, f := range fields {
arr := strings.Split(f, "=")
if len(arr) != 2 {
e = errors.New("Invalid WWW-Auth field format for " + f)
return
}
key := arr[0]
value := strings.Replace(arr[1], `"`, "", -1)
fieldMap[key] = value
}
return
}

// GetImageAllData extracts content info from each pulled image. Currently it gets system package info.
func GetImageAllData(pulledImages ImageSet) (outMapMap map[string]map[string]interface{}) {
//Map ImageID -> Script Map; Script Map: Script name -> output
Expand Down

0 comments on commit 83374a9

Please sign in to comment.