Skip to content

Commit

Permalink
Add image watcher
Browse files Browse the repository at this point in the history
  • Loading branch information
nakabonne committed Dec 2, 2020
1 parent f8e8b7e commit 23f7fb3
Show file tree
Hide file tree
Showing 20 changed files with 539 additions and 51 deletions.
5 changes: 5 additions & 0 deletions go.mod
Expand Up @@ -9,6 +9,9 @@ require (
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46
github.com/aws/aws-sdk-go v1.34.5 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/envoyproxy/protoc-gen-validate v0.1.0
github.com/fsouza/fake-gcs-server v1.21.0
github.com/golang/mock v1.4.4
Expand All @@ -19,6 +22,8 @@ require (
github.com/hashicorp/golang-lru v0.5.1
github.com/klauspost/compress v1.10.11 // indirect
github.com/minio/minio-go/v7 v7.0.5
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/prometheus/client_golang v1.6.0
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.9.1
Expand Down
17 changes: 17 additions & 0 deletions go.sum
Expand Up @@ -96,6 +96,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
Expand Down Expand Up @@ -268,6 +274,7 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
Expand All @@ -293,6 +300,7 @@ github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
Expand Down Expand Up @@ -334,6 +342,10 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
Expand All @@ -347,6 +359,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A=
github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
Expand All @@ -356,10 +369,12 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI=
github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
Expand All @@ -372,6 +387,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
Expand Down Expand Up @@ -545,6 +561,7 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
15 changes: 15 additions & 0 deletions pkg/app/piped/imageprovider/BUILD.bazel
@@ -0,0 +1,15 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["provider.go"],
importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider",
visibility = ["//visibility:public"],
deps = [
"//pkg/app/piped/imageprovider/gcr:go_default_library",
"//pkg/config:go_default_library",
"//pkg/model:go_default_library",
"@com_github_docker_distribution//registry/client/auth/challenge:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
9 changes: 9 additions & 0 deletions pkg/app/piped/imageprovider/gcr/BUILD.bazel
Expand Up @@ -5,4 +5,13 @@ go_library(
srcs = ["gcr.go"],
importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/gcr",
visibility = ["//visibility:public"],
deps = [
"//pkg/config:go_default_library",
"//pkg/model:go_default_library",
"@com_github_docker_distribution//registry/client:go_default_library",
"@com_github_docker_distribution//registry/client/auth:go_default_library",
"@com_github_docker_distribution//registry/client/auth/challenge:go_default_library",
"@com_github_docker_distribution//registry/client/transport:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
90 changes: 90 additions & 0 deletions pkg/app/piped/imageprovider/gcr/gcr.go
Expand Up @@ -13,3 +13,93 @@
// limitations under the License.

package gcr

import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"time"

"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/docker/distribution/registry/client/transport"
"go.uber.org/zap"

"github.com/pipe-cd/pipe/pkg/config"
"github.com/pipe-cd/pipe/pkg/model"
)

type Provider struct {
name string
baseURL url.URL
transport http.RoundTripper

logger *zap.Logger
}

type determineURL func(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error)

func NewProvider(name string, cfg *config.ImageProviderGCRConfig, fn determineURL, logger *zap.Logger) (*Provider, error) {
var tx http.RoundTripper = &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 10 * time.Second,
Proxy: http.ProxyFromEnvironment,
}
manager := challenge.NewSimpleManager()

u, err := fn(manager, tx, cfg.Address)
if err != nil {
return nil, fmt.Errorf("failed to determine registry URL: %w", err)
}
a := newAuthorizer(tx, manager)
return &Provider{
name: name,
baseURL: *u,
transport: transport.NewTransport(tx, a),
logger: logger.Named("gcr-provider"),
}, nil
}

func (p *Provider) Name() string {
return p.name
}

func (p *Provider) Type() model.ImageProviderType {
return model.ImageProviderTypeGCR
}

func (p *Provider) ParseImage(image string) (*model.ImageName, error) {
ss := strings.SplitN(image, "/", 2)
if len(ss) < 2 {
return nil, fmt.Errorf("invalid image format (e.g. gcr.io/pipecd/helloworld)")
}
return &model.ImageName{
Domain: ss[0],
Repo: ss[1],
}, nil
}

func (p *Provider) GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error) {
repository, err := client.NewRepository(image, p.baseURL.String(), p.transport)
if err != nil {
return nil, err
}
_, err = repository.Tags(ctx).All(ctx)
if err != nil {
return nil, err
}
// TODO: Give back latest image from GCR
return nil, nil
}

func newAuthorizer(tx http.RoundTripper, manager challenge.Manager) transport.RequestModifier {
// TODO: Use credentials for GCR configured by user
authHandlers := []auth.AuthenticationHandler{
auth.NewTokenHandler(tx, nil, "", "pull"),
auth.NewBasicHandler(nil),
}
return auth.NewAuthorizer(manager, authHandlers...)
}
72 changes: 72 additions & 0 deletions pkg/app/piped/imageprovider/provider.go
@@ -0,0 +1,72 @@
// Copyright 2020 The PipeCD Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package imageprovider

import (
"context"
"fmt"
"net/http"
"net/url"

"github.com/docker/distribution/registry/client/auth/challenge"
"go.uber.org/zap"

"github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/gcr"
"github.com/pipe-cd/pipe/pkg/config"
"github.com/pipe-cd/pipe/pkg/model"
)

// Provider acs as a container registry client.
type Provider interface {
// Name gives back the provider name that is unique in the Piped.
Name() string
// Type indicates which container registry client to act as.
Type() model.ImageProviderType
// ParseImage converts the given string into structured image.
ParseImage(image string) (*model.ImageName, error)
// GetLatestImages gives back an image with the latest tag.
GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error)
}

// NewProvider yields an appropriate provider according to the given config.
func NewProvider(cfg *config.PipedImageProvider, logger *zap.Logger) (Provider, error) {
switch cfg.Type {
case model.ImageProviderTypeGCR:
return gcr.NewProvider(cfg.Name, cfg.GCRConfig, doChallenge, logger)
case model.ImageProviderTypeDockerhub:
return nil, fmt.Errorf("not implemented yet")
case model.ImageProviderTypeECR:
return nil, fmt.Errorf("not implemented yet")
default:
return nil, fmt.Errorf("unknown image provider type: %s", cfg.Type)
}
}

func doChallenge(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error) {
registryURL := url.URL{
Scheme: "https",
Host: domain,
Path: "/v2/",
}
cs, err := manager.GetChallenges(registryURL)
if err != nil {
return nil, err
}
if len(cs) == 0 {
// TODO: Handle referring to https://github.com/fluxcd/flux/blob/72743f209207453a4326757ba89fb03cb514b34d/pkg/registry/client_factory.go#L64-L91
}

return &registryURL, nil
}
7 changes: 6 additions & 1 deletion pkg/app/piped/imagewatcher/BUILD.bazel
Expand Up @@ -5,5 +5,10 @@ go_library(
srcs = ["watcher.go"],
importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imagewatcher",
visibility = ["//visibility:public"],
deps = ["@org_uber_go_zap//:go_default_library"],
deps = [
"//pkg/app/piped/imageprovider:go_default_library",
"//pkg/config:go_default_library",
"//pkg/git:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)

0 comments on commit 23f7fb3

Please sign in to comment.