diff --git a/Gopkg.lock b/Gopkg.lock index 35238780..638fa29e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -363,6 +363,14 @@ pruneopts = "NUT" revision = "30785a2c434e431ef7c507b54617d6a951d5f2b4" +[[projects]] + branch = "master" + digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239" + name = "golang.org/x/sync" + packages = ["errgroup"] + pruneopts = "NUT" + revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + [[projects]] digest = "1:a969921c213e6d90554219689f484043e1e897d1936e524cfbb4fd6cb8ae2736" name = "golang.org/x/sys" @@ -782,6 +790,7 @@ "github.com/mattbaird/jsonpatch", "github.com/sergi/go-diff/diffmatchpatch", "go.uber.org/zap", + "golang.org/x/sync/errgroup", "k8s.io/api/admission/v1beta1", "k8s.io/api/admissionregistration/v1beta1", "k8s.io/api/core/v1", diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 120c9312..c627257f 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -122,19 +122,21 @@ func main() { go kubeInformerFactory.Start(stopCh) go buildInformerFactory.Start(stopCh) + go cachingInformerFactory.Start(stopCh) for i, synced := range []cache.InformerSynced{ buildInformer.Informer().HasSynced, buildTemplateInformer.Informer().HasSynced, clusterBuildTemplateInformer.Informer().HasSynced, - // TODO(mattmoor): Start waiting for sync after something sets up an event otherwise it hangs here. - // imageInformer.Informer().HasSynced, + imageInformer.Informer().HasSynced, } { if ok := cache.WaitForCacheSync(stopCh, synced); !ok { logger.Fatalf("failed to wait for cache at index %v to sync", i) } } + logger.Infof("SYNCED") + // Start all of the controllers. for _, ctrlr := range controllers { go func(ctrlr controller.Interface) { diff --git a/config/200-clusterrole.yaml b/config/200-clusterrole.yaml index 82360cfb..97e67c50 100644 --- a/config/200-clusterrole.yaml +++ b/config/200-clusterrole.yaml @@ -20,4 +20,4 @@ rules: verbs: ["get", "list", "create", "update", "delete", "patch", "watch"] - apiGroups: ["caching.internal.knative.dev"] resources: ["images"] - verbs: ["get", "list", "create", "update", "delete", "patch", "watch"] + verbs: ["get", "list", "create", "update", "delete", "deletecollection", "patch", "watch"] diff --git a/pkg/reconciler/buildtemplate/buildtemplate.go b/pkg/reconciler/buildtemplate/buildtemplate.go index 0024cebb..1ef36a37 100644 --- a/pkg/reconciler/buildtemplate/buildtemplate.go +++ b/pkg/reconciler/buildtemplate/buildtemplate.go @@ -20,19 +20,25 @@ import ( "context" "go.uber.org/zap" + "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/cache" "github.com/knative/pkg/controller" + "github.com/knative/pkg/kmeta" "github.com/knative/pkg/logging" "github.com/knative/pkg/logging/logkey" + "github.com/knative/build/pkg/apis/build/v1alpha1" clientset "github.com/knative/build/pkg/client/clientset/versioned" buildscheme "github.com/knative/build/pkg/client/clientset/versioned/scheme" informers "github.com/knative/build/pkg/client/informers/externalversions/build/v1alpha1" listers "github.com/knative/build/pkg/client/listers/build/v1alpha1" + "github.com/knative/build/pkg/reconciler/buildtemplate/resources" + caching "github.com/knative/caching/pkg/apis/caching/v1alpha1" cachingclientset "github.com/knative/caching/pkg/client/clientset/versioned" cachinginformers "github.com/knative/caching/pkg/client/informers/externalversions/caching/v1alpha1" cachinglisters "github.com/knative/caching/pkg/client/listers/caching/v1alpha1" @@ -50,7 +56,7 @@ type Reconciler struct { cachingclientset cachingclientset.Interface buildTemplatesLister listers.BuildTemplateLister - imageLister cachinglisters.ImageLister + imagesLister cachinglisters.ImageLister // Sugared logger is easier to use but is not as performant as the // raw logger. In performance critical paths, call logger.Desugar() @@ -87,7 +93,7 @@ func NewController( buildclientset: buildclientset, cachingclientset: cachingclientset, buildTemplatesLister: buildTemplateInformer.Lister(), - imageLister: imageInformer.Lister(), + imagesLister: imageInformer.Lister(), Logger: logger, } impl := controller.NewImpl(r, logger, "BuildTemplates") @@ -99,6 +105,14 @@ func NewController( UpdateFunc: controller.PassNew(impl.Enqueue), }) + imageInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: controller.Filter(v1alpha1.SchemeGroupVersion.WithKind("BuildTemplate")), + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: impl.EnqueueControllerOf, + UpdateFunc: controller.PassNew(impl.EnqueueControllerOf), + }, + }) + return impl } @@ -114,7 +128,8 @@ func (c *Reconciler) Reconcile(ctx context.Context, key string) error { } // Get the BuildTemplate resource with this namespace/name - if _, err := c.buildTemplatesLister.BuildTemplates(namespace).Get(name); errors.IsNotFound(err) { + bt, err := c.buildTemplatesLister.BuildTemplates(namespace).Get(name) + if errors.IsNotFound(err) { // The BuildTemplate resource may no longer exist, in which case we stop processing. logger.Errorf("buildtemplate %q in work queue no longer exists", key) return nil @@ -122,6 +137,59 @@ func (c *Reconciler) Reconcile(ctx context.Context, key string) error { return err } - // TODO: Meaningful reconciliation. + if err := c.reconcileImageCaches(ctx, bt); err != nil { + return err + } + return nil } + +func (c *Reconciler) reconcileImageCaches(ctx context.Context, bt *v1alpha1.BuildTemplate) error { + ics := resources.MakeImageCaches(bt) + + eics, err := c.imagesLister.Images(bt.Namespace).List(kmeta.MakeVersionLabelSelector(bt)) + if err != nil { + return err + } + + // Make sure we have all of the desired caching resources. + if missing := allCached(ics, eics); len(missing) > 0 { + grp, _ := errgroup.WithContext(ctx) + + for _, m := range missing { + m := m + grp.Go(func() error { + _, err := c.cachingclientset.CachingV1alpha1().Images(m.Namespace).Create(&m) + return err + }) + } + + // Wait for the creates to complete. + if err := grp.Wait(); err != nil { + return err + } + } + + // Delete any Image caches relevant to older versions of this resource. + propPolicy := metav1.DeletePropagationForeground + return c.cachingclientset.CachingV1alpha1().Images(bt.Namespace).DeleteCollection( + &metav1.DeleteOptions{PropagationPolicy: &propPolicy}, + metav1.ListOptions{LabelSelector: kmeta.MakeOldVersionLabelSelector(bt).String()}, + ) +} + +func allCached(desired []caching.Image, observed []*caching.Image) (missing []caching.Image) { + for _, d := range desired { + found := false + for _, o := range observed { + if d.Name == o.Name { + found = true + break + } + } + if !found { + missing = append(missing, d) + } + } + return missing +} diff --git a/third_party/VENDOR-LICENSE b/third_party/VENDOR-LICENSE index e1017f8a..792c7de2 100644 --- a/third_party/VENDOR-LICENSE +++ b/third_party/VENDOR-LICENSE @@ -3500,6 +3500,39 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +=========================================================== +Import: github.com/knative/build/vendor/golang.org/x/sync + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + =========================================================== Import: github.com/knative/build/vendor/golang.org/x/sys diff --git a/vendor/golang.org/x/sync/AUTHORS b/vendor/golang.org/x/sync/AUTHORS new file mode 100644 index 00000000..15167cd7 --- /dev/null +++ b/vendor/golang.org/x/sync/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/vendor/golang.org/x/sync/CONTRIBUTORS b/vendor/golang.org/x/sync/CONTRIBUTORS new file mode 100644 index 00000000..1c4577e9 --- /dev/null +++ b/vendor/golang.org/x/sync/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/vendor/golang.org/x/sync/LICENSE b/vendor/golang.org/x/sync/LICENSE new file mode 100644 index 00000000..6a66aea5 --- /dev/null +++ b/vendor/golang.org/x/sync/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/sync/PATENTS b/vendor/golang.org/x/sync/PATENTS new file mode 100644 index 00000000..73309904 --- /dev/null +++ b/vendor/golang.org/x/sync/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go new file mode 100644 index 00000000..533438d9 --- /dev/null +++ b/vendor/golang.org/x/sync/errgroup/errgroup.go @@ -0,0 +1,67 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package errgroup provides synchronization, error propagation, and Context +// cancelation for groups of goroutines working on subtasks of a common task. +package errgroup + +import ( + "sync" + + "golang.org/x/net/context" +) + +// A Group is a collection of goroutines working on subtasks that are part of +// the same overall task. +// +// A zero Group is valid and does not cancel on error. +type Group struct { + cancel func() + + wg sync.WaitGroup + + errOnce sync.Once + err error +} + +// WithContext returns a new Group and an associated Context derived from ctx. +// +// The derived Context is canceled the first time a function passed to Go +// returns a non-nil error or the first time Wait returns, whichever occurs +// first. +func WithContext(ctx context.Context) (*Group, context.Context) { + ctx, cancel := context.WithCancel(ctx) + return &Group{cancel: cancel}, ctx +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the first non-nil error (if any) from them. +func (g *Group) Wait() error { + g.wg.Wait() + if g.cancel != nil { + g.cancel() + } + return g.err +} + +// Go calls the given function in a new goroutine. +// +// The first call to return a non-nil error cancels the group; its error will be +// returned by Wait. +func (g *Group) Go(f func() error) { + g.wg.Add(1) + + go func() { + defer g.wg.Done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + g.err = err + if g.cancel != nil { + g.cancel() + } + }) + } + }() +}