From 23f7fb37f6446d129d1d201753ebbf2f41ef9ed7 Mon Sep 17 00:00:00 2001 From: nakabonne Date: Thu, 26 Nov 2020 18:05:13 +0900 Subject: [PATCH] Add image watcher --- go.mod | 5 + go.sum | 17 +++ pkg/app/piped/imageprovider/BUILD.bazel | 15 ++ pkg/app/piped/imageprovider/gcr/BUILD.bazel | 9 ++ pkg/app/piped/imageprovider/gcr/gcr.go | 90 ++++++++++++ pkg/app/piped/imageprovider/provider.go | 72 ++++++++++ pkg/app/piped/imagewatcher/BUILD.bazel | 7 +- pkg/app/piped/imagewatcher/watcher.go | 144 +++++++++++++++---- pkg/config/BUILD.bazel | 1 + pkg/config/config.go | 7 + pkg/config/deployment.go | 15 -- pkg/config/deployment_kubernetes.go | 2 - pkg/config/image_watcher.go | 37 +++++ pkg/config/piped.go | 22 ++- pkg/config/piped_test.go | 23 ++- pkg/config/testdata/.pipe/image-watcher.yaml | 12 ++ pkg/config/testdata/piped/piped-config.yaml | 13 ++ pkg/model/BUILD.bazel | 1 + pkg/model/image_name.go | 67 +++++++++ repositories.bzl | 31 ++++ 20 files changed, 539 insertions(+), 51 deletions(-) create mode 100644 pkg/app/piped/imageprovider/BUILD.bazel create mode 100644 pkg/app/piped/imageprovider/provider.go create mode 100644 pkg/config/image_watcher.go create mode 100644 pkg/config/testdata/.pipe/image-watcher.yaml create mode 100644 pkg/model/image_name.go diff --git a/go.mod b/go.mod index 289114f389..96e94689c7 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,9 @@ require ( github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 github.com/aws/aws-sdk-go v1.34.5 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/docker/distribution v2.7.1+incompatible + github.com/docker/go-metrics v0.0.1 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/envoyproxy/protoc-gen-validate v0.1.0 github.com/fsouza/fake-gcs-server v1.21.0 github.com/golang/mock v1.4.4 @@ -19,6 +22,8 @@ require ( github.com/hashicorp/golang-lru v0.5.1 github.com/klauspost/compress v1.10.11 // indirect github.com/minio/minio-go/v7 v7.0.5 + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect github.com/prometheus/client_golang v1.6.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 diff --git a/go.sum b/go.sum index f53eb46d6e..afa018c240 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -268,6 +274,7 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= @@ -293,6 +300,7 @@ github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -334,6 +342,10 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -347,6 +359,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -356,10 +369,12 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -372,6 +387,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -545,6 +561,7 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/app/piped/imageprovider/BUILD.bazel b/pkg/app/piped/imageprovider/BUILD.bazel new file mode 100644 index 0000000000..090250a5c1 --- /dev/null +++ b/pkg/app/piped/imageprovider/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["provider.go"], + importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider", + visibility = ["//visibility:public"], + deps = [ + "//pkg/app/piped/imageprovider/gcr:go_default_library", + "//pkg/config:go_default_library", + "//pkg/model:go_default_library", + "@com_github_docker_distribution//registry/client/auth/challenge:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], +) diff --git a/pkg/app/piped/imageprovider/gcr/BUILD.bazel b/pkg/app/piped/imageprovider/gcr/BUILD.bazel index 385cef7a25..530e074bd3 100644 --- a/pkg/app/piped/imageprovider/gcr/BUILD.bazel +++ b/pkg/app/piped/imageprovider/gcr/BUILD.bazel @@ -5,4 +5,13 @@ go_library( srcs = ["gcr.go"], importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/gcr", visibility = ["//visibility:public"], + deps = [ + "//pkg/config:go_default_library", + "//pkg/model:go_default_library", + "@com_github_docker_distribution//registry/client:go_default_library", + "@com_github_docker_distribution//registry/client/auth:go_default_library", + "@com_github_docker_distribution//registry/client/auth/challenge:go_default_library", + "@com_github_docker_distribution//registry/client/transport:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], ) diff --git a/pkg/app/piped/imageprovider/gcr/gcr.go b/pkg/app/piped/imageprovider/gcr/gcr.go index 6392f3566d..5c87fffde7 100644 --- a/pkg/app/piped/imageprovider/gcr/gcr.go +++ b/pkg/app/piped/imageprovider/gcr/gcr.go @@ -13,3 +13,93 @@ // limitations under the License. package gcr + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/docker/distribution/registry/client" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/docker/distribution/registry/client/transport" + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/model" +) + +type Provider struct { + name string + baseURL url.URL + transport http.RoundTripper + + logger *zap.Logger +} + +type determineURL func(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error) + +func NewProvider(name string, cfg *config.ImageProviderGCRConfig, fn determineURL, logger *zap.Logger) (*Provider, error) { + var tx http.RoundTripper = &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + Proxy: http.ProxyFromEnvironment, + } + manager := challenge.NewSimpleManager() + + u, err := fn(manager, tx, cfg.Address) + if err != nil { + return nil, fmt.Errorf("failed to determine registry URL: %w", err) + } + a := newAuthorizer(tx, manager) + return &Provider{ + name: name, + baseURL: *u, + transport: transport.NewTransport(tx, a), + logger: logger.Named("gcr-provider"), + }, nil +} + +func (p *Provider) Name() string { + return p.name +} + +func (p *Provider) Type() model.ImageProviderType { + return model.ImageProviderTypeGCR +} + +func (p *Provider) ParseImage(image string) (*model.ImageName, error) { + ss := strings.SplitN(image, "/", 2) + if len(ss) < 2 { + return nil, fmt.Errorf("invalid image format (e.g. gcr.io/pipecd/helloworld)") + } + return &model.ImageName{ + Domain: ss[0], + Repo: ss[1], + }, nil +} + +func (p *Provider) GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error) { + repository, err := client.NewRepository(image, p.baseURL.String(), p.transport) + if err != nil { + return nil, err + } + _, err = repository.Tags(ctx).All(ctx) + if err != nil { + return nil, err + } + // TODO: Give back latest image from GCR + return nil, nil +} + +func newAuthorizer(tx http.RoundTripper, manager challenge.Manager) transport.RequestModifier { + // TODO: Use credentials for GCR configured by user + authHandlers := []auth.AuthenticationHandler{ + auth.NewTokenHandler(tx, nil, "", "pull"), + auth.NewBasicHandler(nil), + } + return auth.NewAuthorizer(manager, authHandlers...) +} diff --git a/pkg/app/piped/imageprovider/provider.go b/pkg/app/piped/imageprovider/provider.go new file mode 100644 index 0000000000..652b0c80cb --- /dev/null +++ b/pkg/app/piped/imageprovider/provider.go @@ -0,0 +1,72 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imageprovider + +import ( + "context" + "fmt" + "net/http" + "net/url" + + "github.com/docker/distribution/registry/client/auth/challenge" + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/gcr" + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/model" +) + +// Provider acs as a container registry client. +type Provider interface { + // Name gives back the provider name that is unique in the Piped. + Name() string + // Type indicates which container registry client to act as. + Type() model.ImageProviderType + // ParseImage converts the given string into structured image. + ParseImage(image string) (*model.ImageName, error) + // GetLatestImages gives back an image with the latest tag. + GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error) +} + +// NewProvider yields an appropriate provider according to the given config. +func NewProvider(cfg *config.PipedImageProvider, logger *zap.Logger) (Provider, error) { + switch cfg.Type { + case model.ImageProviderTypeGCR: + return gcr.NewProvider(cfg.Name, cfg.GCRConfig, doChallenge, logger) + case model.ImageProviderTypeDockerhub: + return nil, fmt.Errorf("not implemented yet") + case model.ImageProviderTypeECR: + return nil, fmt.Errorf("not implemented yet") + default: + return nil, fmt.Errorf("unknown image provider type: %s", cfg.Type) + } +} + +func doChallenge(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error) { + registryURL := url.URL{ + Scheme: "https", + Host: domain, + Path: "/v2/", + } + cs, err := manager.GetChallenges(registryURL) + if err != nil { + return nil, err + } + if len(cs) == 0 { + // TODO: Handle referring to https://github.com/fluxcd/flux/blob/72743f209207453a4326757ba89fb03cb514b34d/pkg/registry/client_factory.go#L64-L91 + } + + return ®istryURL, nil +} diff --git a/pkg/app/piped/imagewatcher/BUILD.bazel b/pkg/app/piped/imagewatcher/BUILD.bazel index deb8ad750d..11fa942c48 100644 --- a/pkg/app/piped/imagewatcher/BUILD.bazel +++ b/pkg/app/piped/imagewatcher/BUILD.bazel @@ -5,5 +5,10 @@ go_library( srcs = ["watcher.go"], importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imagewatcher", visibility = ["//visibility:public"], - deps = ["@org_uber_go_zap//:go_default_library"], + deps = [ + "//pkg/app/piped/imageprovider:go_default_library", + "//pkg/config:go_default_library", + "//pkg/git:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], ) diff --git a/pkg/app/piped/imagewatcher/watcher.go b/pkg/app/piped/imagewatcher/watcher.go index c7a86fb350..fd7240e8a1 100644 --- a/pkg/app/piped/imagewatcher/watcher.go +++ b/pkg/app/piped/imagewatcher/watcher.go @@ -13,68 +13,160 @@ // limitations under the License. // Package imagewatcher provides a piped component -// that periodically checks the image registry and updates +// that periodically checks the container registry and updates // the image if there are differences with Git. package imagewatcher import ( "context" + "sync" "time" "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider" + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/git" ) type Watcher interface { Run(context.Context) error } -type watcher struct { - timer *time.Timer - logger *zap.Logger +type gitClient interface { + Clone(ctx context.Context, repoID, remote, branch, destination string) (git.Repo, error) } -type imageRepos map[string]imageRepo -type imageRepo struct { +type watcher struct { + config *config.PipedSpec + gitClient gitClient + logger *zap.Logger + + mu sync.RWMutex + // Indexed by repo id. + gitRepos map[string]git.Repo } -func NewWatcher(interval time.Duration, logger *zap.Logger) Watcher { +func NewWatcher(cfg *config.PipedSpec, gitClient gitClient, logger *zap.Logger) Watcher { return &watcher{ - timer: time.NewTimer(interval), - logger: logger, + config: cfg, + gitClient: gitClient, + logger: logger.Named("image-watcher"), } } +// Run spawns goroutines for each image provider. func (w *watcher) Run(ctx context.Context) error { + // Pre-clone to cache the registered git repositories. + w.gitRepos = make(map[string]git.Repo, len(w.config.Repositories)) + for _, r := range w.config.Repositories { + repo, err := w.gitClient.Clone(ctx, r.RepoID, r.Remote, r.Branch, "") + if err != nil { + w.logger.Error("failed to clone repository", + zap.String("repo-id", r.RepoID), + zap.Error(err), + ) + return err + } + w.gitRepos[r.RepoID] = repo + } + + for _, cfg := range w.config.ImageProviders { + p, err := imageprovider.NewProvider(&cfg, w.logger) + if err != nil { + return err + } + + go w.run(ctx, p, cfg.PullInterval.Duration()) + } + return nil +} + +// run periodically compares the image stored in the given provider and one stored in git. +// And then pushes those with differences. +func (w *watcher) run(ctx context.Context, provider imageprovider.Provider, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { select { case <-ctx.Done(): - return nil - case <-w.timer.C: - reposInReg := w.fetchFromRegistry() - reposInGit := w.fetchFromGit() - outdated := calculateChanges(reposInReg, reposInGit) - if err := w.update(outdated); err != nil { + return + case <-ticker.C: + targets := make([]config.ImageWatcherTarget, 0) + // Collect target images for each git repository. + for id, repo := range w.gitRepos { + w.mu.RLock() + branch := repo.GetClonedBranch() + w.mu.RUnlock() + if err := repo.Pull(ctx, branch); err != nil { + w.logger.Error("failed to update repository branch", + zap.String("repo-id", id), + zap.Error(err), + ) + continue + } + + cfg, ok, err := config.LoadImageWatcher(repo.GetPath()) + if err != nil { + w.logger.Error("failed to load configuration file for Image Watcher", zap.Error(err)) + continue + } + if !ok { + w.logger.Error("configuration file for Image Watcher not found", zap.Error(err)) + continue + } + t := filterTargets(provider.Name(), cfg.Targets) + targets = append(targets, t...) + } + + outdated, err := determineUpdates(ctx, targets, provider) + if err != nil { + w.logger.Error("failed to determine which one should be updated", zap.Error(err)) + continue + } + if len(outdated) == 0 { + w.logger.Info("no image to be updated") + continue + } + if err := update(outdated); err != nil { w.logger.Error("failed to update image", zap.Error(err)) + continue } } } - return nil } -func (w *watcher) fetchFromRegistry() imageRepos { - return nil +// filterTargets gives back the targets corresponding to the given provider. +func filterTargets(provider string, targets []config.ImageWatcherTarget) (filtered []config.ImageWatcherTarget) { + for _, t := range targets { + if t.Provider == provider { + filtered = append(filtered, t) + } + } + return } -func (w *watcher) fetchFromGit() imageRepos { - return nil -} +// determineUpdates gives back target images to be updated. +func determineUpdates(ctx context.Context, targets []config.ImageWatcherTarget, provider imageprovider.Provider) (outdated []config.ImageWatcherTarget, err error) { + for _, target := range targets { + i, err := provider.ParseImage(target.Image) + if err != nil { + return nil, err + } + // TODO: Control not to reach the rate limit + _, err = provider.GetLatestImage(ctx, i) + if err != nil { + return nil, err + } + // TODO: Compares between image repos in the image registry and image repos in git + // And then gives back image repos to be updated. + } -func (w *watcher) update(targets imageRepos) error { - return nil + return } -// calculateChanges compares between image repos in the image registry and -// image repos in git. And then gives back image repos to be updated. -func calculateChanges(x, y imageRepos) imageRepos { +func update(targets []config.ImageWatcherTarget) error { + // TODO: Make it possible to push outdated images to Git return nil } diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel index 78d1857624..49e90326f3 100644 --- a/pkg/config/BUILD.bazel +++ b/pkg/config/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "deployment_lambda.go", "deployment_terraform.go", "duration.go", + "image_watcher.go", "piped.go", "replicas.go", "sealed_secret.go", diff --git a/pkg/config/config.go b/pkg/config/config.go index 3396f72a93..409d4a31db 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -61,6 +61,8 @@ const ( // This configuration file should be placed in .pipe directory // at the root of the repository. KindAnalysisTemplate Kind = "AnalysisTemplate" + // KindImageWatcher represents configuration for Repo Watcher. + KindImageWatcher Kind = "ImageWatcher" ) // Config represents configuration data load from file. @@ -79,6 +81,7 @@ type Config struct { PipedSpec *PipedSpec ControlPlaneSpec *ControlPlaneSpec AnalysisTemplateSpec *AnalysisTemplateSpec + ImageWatcherSpec *ImageWatcherSpec SealedSecretSpec *SealedSecretSpec } @@ -138,6 +141,10 @@ func (c *Config) init(kind Kind, apiVersion string) error { c.SealedSecretSpec = &SealedSecretSpec{} c.spec = c.SealedSecretSpec + case KindImageWatcher: + c.ImageWatcherSpec = &ImageWatcherSpec{} + c.spec = c.ImageWatcherSpec + default: return fmt.Errorf("unsupported kind: %s", c.Kind) } diff --git a/pkg/config/deployment.go b/pkg/config/deployment.go index 8ed507498f..cfe79f9d24 100644 --- a/pkg/config/deployment.go +++ b/pkg/config/deployment.go @@ -263,21 +263,6 @@ type TemplatableAnalysisHTTP struct { Template AnalysisTemplateRef `json:"template"` } -type DeploymentImageWatcher struct { - Targets []ImageWatcherTarget `json:"targets"` -} - -type ImageWatcherTarget struct { - Provider string `json:"provider"` - Image string `json:"image"` - Path ImageWatcherTargetPath `json:"path"` -} - -type ImageWatcherTargetPath struct { - Filename string `json:"filename"` - Field string `json:"field"` -} - type SealedSecretMapping struct { // Relative path from the application directory to sealed secret file. Path string `json:"path"` diff --git a/pkg/config/deployment_kubernetes.go b/pkg/config/deployment_kubernetes.go index bd34f8cf9e..2da86ddfac 100644 --- a/pkg/config/deployment_kubernetes.go +++ b/pkg/config/deployment_kubernetes.go @@ -34,8 +34,6 @@ type KubernetesDeploymentSpec struct { Workloads []K8sResourceReference `json:"workloads"` // Which method should be used for traffic routing. TrafficRouting *KubernetesTrafficRouting `json:"trafficRouting"` - // Configuration for automatic image updates. - ImageWatcher DeploymentImageWatcher `json:"imageWatcher"` } // Validate returns an error if any wrong configuration value was found. diff --git a/pkg/config/image_watcher.go b/pkg/config/image_watcher.go new file mode 100644 index 0000000000..c6ffef2ea5 --- /dev/null +++ b/pkg/config/image_watcher.go @@ -0,0 +1,37 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +type ImageWatcherSpec struct { + Targets []ImageWatcherTarget `json:"targets"` +} + +type ImageWatcherTarget struct { + Provider string `json:"provider"` + Image string `json:"image"` + FilePath string `json:"filePath"` + Field string `json:"field"` +} + +// LoadImageWatcher finds the config file for the image watcher in the .pipe directory first up. +// And returns parsed config, False is returned as the second returned value if not found. +func LoadImageWatcher(repoRoot string) (*ImageWatcherSpec, bool, error) { + // TODO: Load image watcher config, referring to AnalysisTemplateSpec + return nil, false, nil +} + +func (s *ImageWatcherSpec) Validate() error { + return nil +} diff --git a/pkg/config/piped.go b/pkg/config/piped.go index 3164590259..41792f71b6 100644 --- a/pkg/config/piped.go +++ b/pkg/config/piped.go @@ -374,8 +374,10 @@ type AnalysisProviderStackdriverConfig struct { } type PipedImageProvider struct { - Name string - Type model.ImageProviderType + Name string `json:"name"` + Type model.ImageProviderType `json:"type"` + // Default is five minute. + PullInterval Duration `json:"pullInterval"` DockerhubConfig *ImageProviderDockerhubConfig GCRConfig *ImageProviderGCRConfig @@ -383,9 +385,11 @@ type PipedImageProvider struct { } type genericPipedImageProvider struct { - Name string `json:"name"` - Type model.ImageProviderType `json:"type"` - Config json.RawMessage `json:"config"` + Name string `json:"name"` + Type model.ImageProviderType `json:"type"` + PullInterval Duration `json:"pullInterval"` + + Config json.RawMessage `json:"config"` } func (p *PipedImageProvider) UnmarshalJSON(data []byte) error { @@ -396,6 +400,10 @@ func (p *PipedImageProvider) UnmarshalJSON(data []byte) error { } p.Name = gp.Name p.Type = gp.Type + p.PullInterval = gp.PullInterval + if p.PullInterval == 0 { + p.PullInterval = Duration(time.Minute * 5) + } switch p.Type { case model.ImageProviderTypeDockerhub: @@ -420,6 +428,8 @@ func (p *PipedImageProvider) UnmarshalJSON(data []byte) error { } type ImageProviderGCRConfig struct { + Address string `json:"address"` + CredentialsFile string `json:"credentialsFile"` } type ImageProviderDockerhubConfig struct { @@ -428,6 +438,8 @@ type ImageProviderDockerhubConfig struct { } type ImageProviderECRConfig struct { + Address string `json:"address"` + TokenFile string `json:"tokenFile"` } type Notifications struct { diff --git a/pkg/config/piped_test.go b/pkg/config/piped_test.go index d3a280e946..0563d5d07d 100644 --- a/pkg/config/piped_test.go +++ b/pkg/config/piped_test.go @@ -150,13 +150,32 @@ func TestPipedConfig(t *testing.T) { }, ImageProviders: []PipedImageProvider{ { - Name: "my-dockerhub", - Type: "DOCKERHUB", + Name: "my-dockerhub", + Type: "DOCKERHUB", + PullInterval: Duration(time.Minute * 5), DockerhubConfig: &ImageProviderDockerhubConfig{ Username: "foo", PasswordFile: "/etc/piped-secret/dockerhub-pass", }, }, + { + Name: "my-gcr", + Type: "GCR", + PullInterval: Duration(time.Minute * 5), + GCRConfig: &ImageProviderGCRConfig{ + Address: "asia.gcr.io", + CredentialsFile: "/etc/piped-secret/gcr-service-account", + }, + }, + { + Name: "my-ecr", + Type: "ECR", + PullInterval: Duration(time.Minute * 5), + ECRConfig: &ImageProviderECRConfig{ + Address: "012345678910.dkr.ecr.us-east-1.amazonaws.com", + TokenFile: "/etc/piped-secret/ecr-authorization-token", + }, + }, }, Notifications: Notifications{ Routes: []NotificationRoute{ diff --git a/pkg/config/testdata/.pipe/image-watcher.yaml b/pkg/config/testdata/.pipe/image-watcher.yaml new file mode 100644 index 0000000000..2f83a5c7c5 --- /dev/null +++ b/pkg/config/testdata/.pipe/image-watcher.yaml @@ -0,0 +1,12 @@ +apiVersion: pipecd.dev/v1beta1 +kind: ImageWatcher +spec: + targets: + - image: gcr.io/pipecd/foo + provider: my-gcr + filePath: foo/deployment.yaml + field: spec.containers[0].image + - image: pipecd/bar + provider: my-dockerhub + filePath: bar/deployment.yaml + field: spec.containers[0].image diff --git a/pkg/config/testdata/piped/piped-config.yaml b/pkg/config/testdata/piped/piped-config.yaml index 1d47843730..565f8c9d2e 100644 --- a/pkg/config/testdata/piped/piped-config.yaml +++ b/pkg/config/testdata/piped/piped-config.yaml @@ -80,9 +80,22 @@ spec: imageProviders: - name: my-dockerhub type: DOCKERHUB + pullInterval: 5m config: username: foo passwordFile: /etc/piped-secret/dockerhub-pass + - name: my-gcr + type: GCR + pullInterval: 5m + config: + address: asia.gcr.io + credentialsFile: /etc/piped-secret/gcr-service-account + - name: my-ecr + type: ECR + pullInterval: 5m + config: + address: 012345678910.dkr.ecr.us-east-1.amazonaws.com + tokenFile: /etc/piped-secret/ecr-authorization-token notifications: routes: diff --git a/pkg/model/BUILD.bazel b/pkg/model/BUILD.bazel index 2f7dd0e90f..18e7bc3443 100644 --- a/pkg/model/BUILD.bazel +++ b/pkg/model/BUILD.bazel @@ -52,6 +52,7 @@ go_library( "environment.go", "event.go", "filestore.go", + "image_name.go", "imageprovider.go", "model.go", "piped.go", diff --git a/pkg/model/image_name.go b/pkg/model/image_name.go new file mode 100644 index 0000000000..11fcf51b78 --- /dev/null +++ b/pkg/model/image_name.go @@ -0,0 +1,67 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import "fmt" + +// ImageName represents an untagged image. Note that images may have +// the domain omitted (e.g. Docker Hub). If they only have single path element, +// the prefix `library` is implied. +// +// Examples: +// - alpine +// - library/alpine +// - gcr.io/pipecd/helloworld +type ImageName struct { + Domain string + Repo string +} + +func (i ImageName) String() string { + if i.Repo == "" { + return "" + } + + var host string + if i.Domain != "" { + host = i.Domain + "/" + } + return fmt.Sprintf("%s%s", host, i.Repo) +} + +// Name gives back just repository name without domain. +func (i ImageName) Name() string { + return i.Repo +} + +// ImageRef represents a tagged image. The tag is allowed to be +// empty, though it is in general undefined what that means +// +// Examples: +// - alpine:3.0 +// - library/alpine:3.0 +// - gcr.io/pipecd/helloworld:0.1.0 +type ImageRef struct { + ImageName + Tag string +} + +func (i ImageRef) String() string { + var tag string + if i.Tag != "" { + tag = ":" + i.Tag + } + return fmt.Sprintf("%s%s", i.ImageName.String(), tag) +} diff --git a/repositories.bzl b/repositories.bzl index 877e656c3a..1d1fedfa7a 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -231,6 +231,25 @@ def go_repositories(): sum = "h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4=", version = "v0.0.0-20181026042036-e10d5fee7954", ) + go_repository( + name = "com_github_docker_distribution", + importpath = "github.com/docker/distribution", + sum = "h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=", + version = "v2.7.1+incompatible", + ) + go_repository( + name = "com_github_docker_go_metrics", + importpath = "github.com/docker/go-metrics", + sum = "h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=", + version = "v0.0.1", + ) + go_repository( + name = "com_github_docker_libtrust", + importpath = "github.com/docker/libtrust", + sum = "h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=", + version = "v0.0.0-20160708172513-aabc10ec26b7", + ) + go_repository( name = "com_github_docker_spdystream", importpath = "github.com/docker/spdystream", @@ -1070,6 +1089,18 @@ def go_repositories(): sum = "h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=", version = "v1.7.0", ) + go_repository( + name = "com_github_opencontainers_go_digest", + importpath = "github.com/opencontainers/go-digest", + sum = "h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=", + version = "v1.0.0", + ) + go_repository( + name = "com_github_opencontainers_image_spec", + importpath = "github.com/opencontainers/image-spec", + sum = "h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=", + version = "v1.0.1", + ) go_repository( name = "com_github_openpeedeep_depguard",