enable caching of remote config requests

cmd/drone-server/config/config.go

@@ -98,7 +98,7 @@ type (
 	}
 
 	Cleanup struct {
-		Disabled  bool         `envconfig:"DRONE_CLEANUP_DISABLED"`
+		Disabled bool          `envconfig:"DRONE_CLEANUP_DISABLED"`
 		Interval time.Duration `envconfig:"DRONE_CLEANUP_INTERVAL"         default:"24h"`
 		Running  time.Duration `envconfig:"DRONE_CLEANUP_DEADLINE_RUNNING" default:"24h"`
 		Pending  time.Duration `envconfig:"DRONE_CLEANUP_DEADLINE_PENDING" default:"24h"`
@@ -188,6 +188,10 @@ type (
 		Filter     []string `envconfig:"DRONE_REPOSITORY_FILTER"`
 		Visibility string   `envconfig:"DRONE_REPOSITORY_VISIBILITY"`
 		Trusted    bool     `envconfig:"DRONE_REPOSITORY_TRUSTED"`
+
+		// THIS SETTING IS INTERNAL USE ONLY AND SHOULD
+		// NOT BE USED OR RELIED UPON IN PRODUCTION.
+		Ignore []string `envconfig:"DRONE_REPOSITORY_IGNORE"`
 	}
 
 	// Registries provides the registry configuration.

cmd/drone-server/inject_plugin.go

@@ -61,10 +61,12 @@ func provideAdmissionPlugin(client *scm.Client, orgs core.OrganizationService, u
 // configuration.
 func provideConfigPlugin(client *scm.Client, contents core.FileService, conf spec.Config) core.ConfigService {
 	return config.Combine(
-		config.Global(
-			conf.Yaml.Endpoint,
-			conf.Yaml.Secret,
-			conf.Yaml.SkipVerify,
+		config.Memoize(
+			config.Global(
+				conf.Yaml.Endpoint,
+				conf.Yaml.Secret,
+				conf.Yaml.SkipVerify,
+			),
 		),
 		config.Repository(contents),
 	)
@@ -80,11 +82,13 @@ func provideConvertPlugin(client *scm.Client, conf spec.Config) core.ConvertServ
 		converter.Jsonnet(
 			conf.Jsonnet.Enabled,
 		),
-		converter.Remote(
-			conf.Convert.Endpoint,
-			conf.Convert.Secret,
-			conf.Convert.Extension,
-			conf.Convert.SkipVerify,
+		converter.Memoize(
+			converter.Remote(
+				conf.Convert.Endpoint,
+				conf.Convert.Secret,
+				conf.Convert.Extension,
+				conf.Convert.SkipVerify,
+			),
 		),
 	)
 }
@@ -130,6 +134,12 @@ func provideValidatePlugin(conf spec.Config) core.ValidateService {
 			conf.Validate.Secret,
 			conf.Validate.SkipVerify,
 		),
+		// THIS FEATURE IS INTERNAL USE ONLY AND SHOULD
+		// NOT BE USED OR RELIED UPON IN PRODUCTION.
+		validator.Filter(
+			nil,
+			conf.Repository.Ignore,
+		),
 	)
 }
 

plugin/config/memoize.go

@@ -0,0 +1,105 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !oss
+
+package config
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/drone/drone/core"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
+)
+
+// cache key pattern used in the cache, comprised of the
+// repository slug and commit sha.
+const keyf = "%d|%s|%s|%s|%s|%s"
+
+// Memoize caches the conversion results for subsequent calls.
+// This micro-optimization is intended for multi-pipeline
+// projects that would otherwise covert the file for each
+// pipeline execution.
+func Memoize(base core.ConfigService) core.ConfigService {
+	// simple cache prevents the same yaml file from being
+	// requested multiple times in a short period.
+	cache, _ := lru.New(10)
+	return &memoize{base: base, cache: cache}
+}
+
+type memoize struct {
+	base  core.ConfigService
+	cache *lru.Cache
+}
+
+func (c *memoize) Find(ctx context.Context, req *core.ConfigArgs) (*core.Config, error) {
+	// this is a minor optimization that prevents caching if the
+	// base converter is a global config service and is disabled.
+	if global, ok := c.base.(*global); ok == true && global.client == nil {
+		return nil, nil
+	}
+
+	// generate the key used to cache the converted file.
+	key := fmt.Sprintf(keyf,
+		req.Repo.ID,
+		req.Build.Event,
+		req.Build.Action,
+		req.Build.Ref,
+		req.Build.After,
+		req.Repo.Config,
+	)
+
+	logger := logrus.WithField("repo", req.Repo.Slug).
+		WithField("build", req.Build.Event).
+		WithField("action", req.Build.Action).
+		WithField("ref", req.Build.Ref).
+		WithField("rev", req.Build.After).
+		WithField("config", req.Repo.Config)
+
+	logger.Trace("extension: configuration: check cache")
+
+	// check the cache for the file and return if exists.
+	cached, ok := c.cache.Get(key)
+	if ok {
+		logger.Trace("extension: configuration: cache hit")
+		return cached.(*core.Config), nil
+	}
+
+	logger.Trace("extension: configuration: cache miss")
+
+	// else find the configuration file.
+	config, err := c.base.Find(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	if config == nil {
+		return nil, nil
+	}
+	if config.Data == "" {
+		return nil, nil
+	}
+
+	// if the configuration file was retrieved
+	// it is temporarily cached. Note that we do
+	// not cache if the commit sha is empty (gogs).
+	if req.Build.After != "" {
+		c.cache.Add(key, config)
+	}
+
+	return config, nil
+}

plugin/config/memoize_oss.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build oss
+
+package config
+
+import (
+	"github.com/drone/drone/core"
+)
+
+// Memoize caches the conversion results for subsequent calls.
+// This micro-optimization is intended for multi-pipeline
+// projects that would otherwise covert the file for each
+// pipeline execution.
+func Memoize(base core.ConvertService) core.ConvertService {
+	return new(noop)
+}

plugin/config/memoize_test.go

@@ -0,0 +1,159 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Drone Non-Commercial License
+// that can be found in the LICENSE file.
+
+// +build !oss
+
+package config
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/drone/drone/core"
+	"github.com/drone/drone/mock"
+
+	"github.com/golang/mock/gomock"
+)
+
+func TestMemoize(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	conf := &core.Config{Data: "{kind: pipeline, type: docker, steps: []}"}
+	args := &core.ConfigArgs{
+		Build:  &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:   &core.Repository{ID: 42},
+		Config: conf,
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	_, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if got, want := service.cache.Len(), 1; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+
+	args.Config = nil // set to nil to prove we get the cached value
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != conf {
+		t.Errorf("Expect result from cache")
+	}
+
+	if got, want := service.cache.Len(), 1; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}
+
+func TestMemoize_Tag(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build:  &core.Build{Ref: "refs/tags/v1.0.0"},
+		Repo:   &core.Repository{ID: 42},
+		Config: &core.Config{Data: "{kind: pipeline, type: docker, steps: []}"},
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != args.Config {
+		t.Errorf("Expect result from cache")
+	}
+}
+
+func TestMemoize_Empty(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build:  &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:   &core.Repository{ID: 42},
+		Config: &core.Config{Data: ""}, // empty
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != nil {
+		t.Errorf("Expect nil response")
+	}
+	if got, want := service.cache.Len(), 0; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}
+
+func TestMemoize_Nil(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build:  &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:   &core.Repository{ID: 42},
+		Config: nil,
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != nil {
+		t.Errorf("Expect nil response")
+	}
+	if got, want := service.cache.Len(), 0; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}
+
+func TestMemoize_Error(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build: &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:  &core.Repository{ID: 42},
+	}
+
+	want := errors.New("not found")
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(nil, want)
+
+	service := Memoize(base).(*memoize)
+	_, err := service.Find(noContext, args)
+	if err == nil {
+		t.Errorf("Expect error from base returned to caller")
+		return
+	}
+	if got, want := service.cache.Len(), 0; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}