introduce ResourceResolver to accept remote resources

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>

cli/options.go

@@ -17,6 +17,7 @@
 package cli
 
 import (
+	"context"
 	"io"
 	"os"
 	"path/filepath"
@@ -35,6 +36,8 @@ import (
 
 // ProjectOptions provides common configuration for loading a project.
 type ProjectOptions struct {
+	ctx context.Context
+
 	// Name is a valid Compose project name to be used or empty.
 	//
 	// If empty, the project loader will automatically infer a reasonable
@@ -301,6 +304,24 @@ func WithResolvedPaths(resolve bool) ProjectOptionsFn {
 	}
 }
 
+// WithContext sets the context used to load model and resources
+func WithContext(ctx context.Context) ProjectOptionsFn {
+	return func(o *ProjectOptions) error {
+		o.ctx = ctx
+		return nil
+	}
+}
+
+// WithResourceLoader register support for ResourceLoader to manage remote resources
+func WithResourceLoader(r loader.ResourceLoader) ProjectOptionsFn {
+	return func(o *ProjectOptions) error {
+		o.loadOptions = append(o.loadOptions, func(options *loader.Options) {
+			options.ResourceLoaders = append(options.ResourceLoaders, r)
+		})
+		return nil
+	}
+}
+
 // DefaultFileNames defines the Compose file names for auto-discovery (in order of preference)
 var DefaultFileNames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"}
 
@@ -367,7 +388,12 @@ func ProjectFromOptions(options *ProjectOptions) (*types.Project, error) {
 		withNamePrecedenceLoad(absWorkingDir, options),
 		withConvertWindowsPaths(options))
 
-	project, err := loader.Load(types.ConfigDetails{
+	ctx := options.ctx
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	project, err := loader.LoadWithContext(ctx, types.ConfigDetails{
 		ConfigFiles: configs,
 		WorkingDir:  workingDir,
 		Environment: options.Environment,

loader/include.go

@@ -17,6 +17,7 @@
 package loader
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -43,12 +44,20 @@ var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{}
 	}
 }
 
-func loadInclude(configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
+func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
 	for _, r := range model.Include {
 		for i, p := range r.Path {
-			if !filepath.IsAbs(p) {
-				r.Path[i] = filepath.Join(configDetails.WorkingDir, p)
+			for _, loader := range options.ResourceLoaders {
+				if loader.Accept(p) {
+					path, err := loader.Load(ctx, p)
+					if err != nil {
+						return nil, err
+					}
+					p = path
+					break
+				}
 			}
+			r.Path[i] = absPath(configDetails.WorkingDir, p)
 		}
 		if r.ProjectDirectory == "" {
 			r.ProjectDirectory = filepath.Dir(r.Path[0])
@@ -65,7 +74,7 @@ func loadInclude(configDetails types.ConfigDetails, model *types.Config, options
 			return nil, err
 		}
 
-		imported, err := load(types.ConfigDetails{
+		imported, err := load(ctx, types.ConfigDetails{
 			WorkingDir:  r.ProjectDirectory,
 			ConfigFiles: types.ToConfigFiles(r.Path),
 			Environment: env,

loader/loader.go

@@ -17,6 +17,7 @@
 package loader
 
 import (
+	"context"
 	"fmt"
 	"os"
 	paths "path"
@@ -68,6 +69,16 @@ type Options struct {
 	projectNameImperativelySet bool
 	// Profiles set profiles to enable
 	Profiles []string
+	// ResourceLoaders manages support for remote resources
+	ResourceLoaders []ResourceLoader
+}
+
+// ResourceLoader is a plugable remote resource resolver
+type ResourceLoader interface {
+	// Accept returns `true` is the resource reference matches ResourceLoader supported protocol(s)
+	Accept(path string) bool
+	// Load returns the path to a local copy of remote resource identified by `path`.
+	Load(ctx context.Context, path string) (string, error)
 }
 
 func (o *Options) clone() *Options {
@@ -85,6 +96,7 @@ func (o *Options) clone() *Options {
 		projectName:                o.projectName,
 		projectNameImperativelySet: o.projectNameImperativelySet,
 		Profiles:                   o.Profiles,
+		ResourceLoaders:            o.ResourceLoaders,
 	}
 }
 
@@ -193,8 +205,14 @@ func parseYAML(source []byte) (map[string]interface{}, PostProcessor, error) {
 	return converted.(map[string]interface{}), &processor, nil
 }
 
-// Load reads a ConfigDetails and returns a fully loaded configuration
+// Load reads a ConfigDetails and returns a fully loaded configuration.
+// Deprecated: use LoadWithContext.
 func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) {
+	return LoadWithContext(context.Background(), configDetails, options...)
+}
+
+// LoadWithContext reads a ConfigDetails and returns a fully loaded configuration
+func LoadWithContext(ctx context.Context, configDetails types.ConfigDetails, options ...func(*Options)) (*types.Project, error) {
 	if len(configDetails.ConfigFiles) < 1 {
 		return nil, errors.Errorf("No files specified")
 	}
@@ -217,10 +235,10 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
 		return nil, err
 	}
 	opts.projectName = projectName
-	return load(configDetails, opts, nil)
+	return load(ctx, configDetails, opts, nil)
 }
 
-func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) {
+func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, loaded []string) (*types.Project, error) {
 	var model *types.Config
 
 	mainFile := configDetails.ConfigFiles[0].Filename
@@ -261,13 +279,13 @@ func load(configDetails types.ConfigDetails, opts *Options, loaded []string) (*t
 
 		configDict = groupXFieldsIntoExtensions(configDict)
 
-		cfg, err := loadSections(file.Filename, configDict, configDetails, opts)
+		cfg, err := loadSections(ctx, file.Filename, configDict, configDetails, opts)
 		if err != nil {
 			return nil, err
 		}
 
 		if !opts.SkipInclude {
-			cfg, err = loadInclude(configDetails, cfg, opts, loaded)
+			cfg, err = loadInclude(ctx, configDetails, cfg, opts, loaded)
 			if err != nil {
 				return nil, err
 			}
@@ -453,7 +471,7 @@ func groupXFieldsIntoExtensions(dict map[string]interface{}) map[string]interfac
 	return dict
 }
 
-func loadSections(filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) {
+func loadSections(ctx context.Context, filename string, config map[string]interface{}, configDetails types.ConfigDetails, opts *Options) (*types.Config, error) {
 	var err error
 	cfg := types.Config{
 		Filename: filename,
@@ -466,7 +484,7 @@ func loadSections(filename string, config map[string]interface{}, configDetails
 		}
 	}
 	cfg.Name = name
-	cfg.Services, err = LoadServices(filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts)
+	cfg.Services, err = LoadServices(ctx, filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -659,7 +677,7 @@ func formatInvalidKeyError(keyPrefix string, key interface{}) error {
 
 // LoadServices produces a ServiceConfig map from a compose file Dict
 // the servicesDict is not validated if directly used. Use Load() to enable validation
-func LoadServices(filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) {
+func LoadServices(ctx context.Context, filename string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options) ([]types.ServiceConfig, error) {
 	var services []types.ServiceConfig
 
 	x, ok := servicesDict[extensions]
@@ -672,7 +690,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
 	}
 
 	for name := range servicesDict {
-		serviceConfig, err := loadServiceWithExtends(filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{})
+		serviceConfig, err := loadServiceWithExtends(ctx, filename, name, servicesDict, workingDir, lookupEnv, opts, &cycleTracker{})
 		if err != nil {
 			return nil, err
 		}
@@ -683,7 +701,7 @@ func LoadServices(filename string, servicesDict map[string]interface{}, workingD
 	return services, nil
 }
 
-func loadServiceWithExtends(filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) {
+func loadServiceWithExtends(ctx context.Context, filename, name string, servicesDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, opts *Options, ct *cycleTracker) (*types.ServiceConfig, error) {
 	if err := ct.Add(filename, name); err != nil {
 		return nil, err
 	}
@@ -707,11 +725,21 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
 		var baseService *types.ServiceConfig
 		file := serviceConfig.Extends.File
 		if file == "" {
-			baseService, err = loadServiceWithExtends(filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct)
+			baseService, err = loadServiceWithExtends(ctx, filename, baseServiceName, servicesDict, workingDir, lookupEnv, opts, ct)
 			if err != nil {
 				return nil, err
 			}
 		} else {
+			for _, loader := range opts.ResourceLoaders {
+				if loader.Accept(file) {
+					path, err := loader.Load(ctx, file)
+					if err != nil {
+						return nil, err
+					}
+					file = path
+					break
+				}
+			}
 			// Resolve the path to the imported file, and load it.
 			baseFilePath := absPath(workingDir, file)
 
@@ -726,7 +754,7 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
 			}
 
 			baseFileServices := getSection(baseFile, "services")
-			baseService, err = loadServiceWithExtends(baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct)
+			baseService, err = loadServiceWithExtends(ctx, baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct)
 			if err != nil {
 				return nil, err
 			}

loader/loader_test.go

@@ -18,6 +18,7 @@ package loader
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -2592,3 +2593,119 @@ services:
 		},
 	})
 }
+
+type customLoader struct {
+	prefix string
+}
+
+func (c customLoader) Accept(s string) bool {
+	return strings.HasPrefix(s, c.prefix+":")
+}
+
+func (c customLoader) Load(ctx context.Context, s string) (string, error) {
+	path := filepath.Join("testdata", c.prefix, s[len(c.prefix)+1:])
+	_, err := os.Stat(path)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Abs(path)
+}
+
+func TestLoadWithRemoteResources(t *testing.T) {
+	config := buildConfigDetails(`
+name: test-remote-resources
+services:
+  foo:
+    extends:
+      file: remote:compose.yaml
+      service: foo
+
+`, nil)
+	p, err := LoadWithContext(context.Background(), config, func(options *Options) {
+		options.SkipConsistencyCheck = true
+		options.SkipNormalization = true
+		options.ResolvePaths = true
+		options.ResourceLoaders = []ResourceLoader{
+			customLoader{prefix: "remote"},
+		}
+	})
+	assert.NilError(t, err)
+	assert.DeepEqual(t, p.Services, types.Services{
+		{
+			Name:        "foo",
+			Image:       "foo",
+			Environment: types.MappingWithEquals{"FOO": strPtr("BAR")},
+			EnvFile: types.StringList{
+				filepath.Join(config.WorkingDir, "testdata", "remote", "env"),
+			},
+			Scale: 1,
+			Volumes: []types.ServiceVolumeConfig{
+				{
+					Type:   types.VolumeTypeBind,
+					Source: filepath.Join(config.WorkingDir, "testdata", "remote"),
+					Target: "/foo",
+					Bind:   &types.ServiceVolumeBind{CreateHostPath: true},
+				},
+			},
+		},
+	})
+}
+
+func TestLoadWithMissingResources(t *testing.T) {
+	config := buildConfigDetails(`
+name: test-missing-resources
+services:
+  foo:
+    extends:
+      file: remote:unavailable.yaml
+      service: foo
+
+`, nil)
+	_, err := LoadWithContext(context.Background(), config, func(options *Options) {
+		options.SkipConsistencyCheck = true
+		options.SkipNormalization = true
+		options.ResolvePaths = true
+		options.ResourceLoaders = []ResourceLoader{
+			customLoader{prefix: "remote"},
+		}
+	})
+	assert.Check(t, os.IsNotExist(err))
+}
+
+func TestLoadWithNestedResources(t *testing.T) {
+	config := buildConfigDetails(`
+name: test-nested-resources
+include: 
+  - remote:nested/compose.yaml
+`, nil)
+	_, err := LoadWithContext(context.Background(), config, func(options *Options) {
+		options.SkipConsistencyCheck = true
+		options.SkipNormalization = true
+		options.ResolvePaths = true
+		options.ResourceLoaders = []ResourceLoader{
+			customLoader{prefix: "remote"},
+		}
+	})
+	assert.NilError(t, err)
+}
+
+func TestLoadWithResourcesCycle(t *testing.T) {
+	config := buildConfigDetails(`
+name: test-resources-cycle
+services:
+  foo:
+    extends:
+      file: remote:cycle/compose.yaml
+      service: foo
+
+`, nil)
+	_, err := LoadWithContext(context.Background(), config, func(options *Options) {
+		options.SkipConsistencyCheck = true
+		options.SkipNormalization = true
+		options.ResolvePaths = true
+		options.ResourceLoaders = []ResourceLoader{
+			customLoader{prefix: "remote"},
+		}
+	})
+	assert.ErrorContains(t, err, "Circular reference")
+}

loader/testdata/remote/compose.yaml

@@ -0,0 +1,7 @@
+services:
+  foo:
+    image: foo
+    env_file:
+      - ./env
+    volumes:
+      - .:/foo

loader/testdata/remote/cycle/compose-cycle.yaml

@@ -0,0 +1,5 @@
+services:
+  bar:
+    extends:
+      file: remote:cycle/compose.yaml
+      service: foo

loader/testdata/remote/cycle/compose.yaml

@@ -0,0 +1,5 @@
+services:
+  foo:
+    extends:
+      file: remote:cycle/compose-cycle.yaml
+      service: bar