Skip to content

Commit

Permalink
Add support for multiple composefile when deploying
Browse files Browse the repository at this point in the history
Using a custom version of mergo to manage special cases

Signed-off-by: Vincent Demeester <vincent@sbr.pm>
  • Loading branch information
vdemeester committed Nov 8, 2017
1 parent ee0615d commit 4809114
Show file tree
Hide file tree
Showing 15 changed files with 1,010 additions and 47 deletions.
6 changes: 3 additions & 3 deletions cli/command/stack/deploy.go
Expand Up @@ -22,7 +22,7 @@ const (

type deployOptions struct {
bundlefile string
composefile string
composefile []string
namespace string
resolveImage string
sendRegistryAuth bool
Expand Down Expand Up @@ -63,9 +63,9 @@ func runDeploy(dockerCli command.Cli, opts deployOptions) error {
}

switch {
case opts.bundlefile == "" && opts.composefile == "":
case opts.bundlefile == "" && len(opts.composefile) == 0:
return errors.Errorf("Please specify either a bundle file (with --bundle-file) or a Compose file (with --compose-file).")
case opts.bundlefile != "" && opts.composefile != "":
case opts.bundlefile != "" && len(opts.composefile) != 0:
return errors.Errorf("You cannot specify both a bundle file and a Compose file.")
case opts.bundlefile != "":
return deployBundle(ctx, dockerCli, opts)
Expand Down
46 changes: 37 additions & 9 deletions cli/command/stack/deploy_composefile.go
Expand Up @@ -38,13 +38,14 @@ func deployCompose(ctx context.Context, dockerCli command.Cli, opts deployOption
return err
}

unsupportedProperties := loader.GetUnsupportedProperties(configDetails)
dicts := getDictsFrom(configDetails.ConfigFiles)
unsupportedProperties := loader.GetUnsupportedProperties(dicts...)
if len(unsupportedProperties) > 0 {
fmt.Fprintf(dockerCli.Err(), "Ignoring unsupported options: %s\n\n",
strings.Join(unsupportedProperties, ", "))
}

deprecatedProperties := loader.GetDeprecatedProperties(configDetails)
deprecatedProperties := loader.GetDeprecatedProperties(dicts...)
if len(deprecatedProperties) > 0 {
fmt.Fprintf(dockerCli.Err(), "Ignoring deprecated options:\n\n%s\n\n",
propertyWarnings(deprecatedProperties))
Expand Down Expand Up @@ -96,6 +97,16 @@ func deployCompose(ctx context.Context, dockerCli command.Cli, opts deployOption
return deployServices(ctx, dockerCli, services, namespace, opts.sendRegistryAuth, opts.resolveImage)
}

func getDictsFrom(configFiles []composetypes.ConfigFile) []map[string]interface{} {
dicts := []map[string]interface{}{}

for _, configFile := range configFiles {
dicts = append(dicts, configFile.Config)
}

return dicts
}

func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
serviceNetworks := map[string]struct{}{}
for _, serviceConfig := range serviceConfigs {
Expand All @@ -119,29 +130,32 @@ func propertyWarnings(properties map[string]string) string {
return strings.Join(msgs, "\n\n")
}

func getConfigDetails(composefile string, stdin io.Reader) (composetypes.ConfigDetails, error) {
func getConfigDetails(composefile []string, stdin io.Reader) (composetypes.ConfigDetails, error) {
var details composetypes.ConfigDetails

if composefile == "-" {
if len(composefile) == 0 {
return details, errors.New("no composefile")
}

if composefile[0] == "-" {
workingDir, err := os.Getwd()
if err != nil {
return details, err
}
details.WorkingDir = workingDir
} else {
absPath, err := filepath.Abs(composefile)
absPath, err := filepath.Abs(composefile[0])
if err != nil {
return details, err
}
details.WorkingDir = filepath.Dir(absPath)
}

configFile, err := getConfigFile(composefile, stdin)
var err error
details.ConfigFiles, err = loadConfigFiles(composefile, stdin)
if err != nil {
return details, err
}
// TODO: support multiple files
details.ConfigFiles = []composetypes.ConfigFile{*configFile}
details.Environment, err = buildEnvironment(os.Environ())
return details, err
}
Expand All @@ -159,7 +173,21 @@ func buildEnvironment(env []string) (map[string]string, error) {
return result, nil
}

func getConfigFile(filename string, stdin io.Reader) (*composetypes.ConfigFile, error) {
func loadConfigFiles(filenames []string, stdin io.Reader) ([]composetypes.ConfigFile, error) {
var configFiles []composetypes.ConfigFile

for _, filename := range filenames {
configFile, err := loadConfigFile(filename, stdin)
if err != nil {
return configFiles, err
}
configFiles = append(configFiles, *configFile)
}

return configFiles, nil
}

func loadConfigFile(filename string, stdin io.Reader) (*composetypes.ConfigFile, error) {
var bytes []byte
var err error

Expand Down
4 changes: 2 additions & 2 deletions cli/command/stack/deploy_composefile_test.go
Expand Up @@ -26,7 +26,7 @@ services:
file := fs.NewFile(t, "test-get-config-details", fs.WithContent(content))
defer file.Remove()

details, err := getConfigDetails(file.Path(), nil)
details, err := getConfigDetails([]string{file.Path()}, nil)
require.NoError(t, err)
assert.Equal(t, filepath.Dir(file.Path()), details.WorkingDir)
require.Len(t, details.ConfigFiles, 1)
Expand All @@ -41,7 +41,7 @@ services:
foo:
image: alpine:3.5
`
details, err := getConfigDetails("-", strings.NewReader(content))
details, err := getConfigDetails([]string{"-"}, strings.NewReader(content))
require.NoError(t, err)
cwd, err := os.Getwd()
require.NoError(t, err)
Expand Down
4 changes: 2 additions & 2 deletions cli/command/stack/opts.go
Expand Up @@ -10,8 +10,8 @@ import (
"github.com/spf13/pflag"
)

func addComposefileFlag(opt *string, flags *pflag.FlagSet) {
flags.StringVarP(opt, "compose-file", "c", "", "Path to a Compose file")
func addComposefileFlag(opt *[]string, flags *pflag.FlagSet) {
flags.StringSliceVarP(opt, "compose-file", "c", []string{}, "Path to a Compose file")
flags.SetAnnotation("compose-file", "version", []string{"1.25"})
}

Expand Down
73 changes: 44 additions & 29 deletions cli/compose/loader/loader.go
Expand Up @@ -45,27 +45,36 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
if len(configDetails.ConfigFiles) < 1 {
return nil, errors.Errorf("No files specified")
}
if len(configDetails.ConfigFiles) > 1 {
return nil, errors.Errorf("Multiple files are not yet supported")
}

configDict := getConfigDict(configDetails)
configDetails.Version = schema.Version(configDict)
configs := []*types.Config{}

if err := validateForbidden(configDict); err != nil {
return nil, err
}
for _, file := range configDetails.ConfigFiles {
configDict := file.Config
configDetails.Version = schema.Version(configDict)

var err error
configDict, err = interpolateConfig(configDict, configDetails.LookupEnv)
if err != nil {
return nil, err
}
if err := validateForbidden(configDict); err != nil {
return nil, err
}

if err := schema.Validate(configDict, configDetails.Version); err != nil {
return nil, err
var err error
configDict, err = interpolateConfig(configDict, configDetails.LookupEnv)
if err != nil {
return nil, err
}

if err := schema.Validate(configDict, configDetails.Version); err != nil {
return nil, err
}

cfg, err := loadSections(configDict, configDetails)
if err != nil {
return nil, err
}

configs = append(configs, cfg)
}
return loadSections(configDict, configDetails)

return merge(configs)
}

func validateForbidden(configDict map[string]interface{}) error {
Expand Down Expand Up @@ -142,14 +151,16 @@ func getSection(config map[string]interface{}, key string) map[string]interface{

// GetUnsupportedProperties returns the list of any unsupported properties that are
// used in the Compose files.
func GetUnsupportedProperties(configDetails types.ConfigDetails) []string {
func GetUnsupportedProperties(configDicts ...map[string]interface{}) []string {
unsupported := map[string]bool{}

for _, service := range getServices(getConfigDict(configDetails)) {
serviceDict := service.(map[string]interface{})
for _, property := range types.UnsupportedProperties {
if _, isSet := serviceDict[property]; isSet {
unsupported[property] = true
for _, configDict := range configDicts {
for _, service := range getServices(configDict) {
serviceDict := service.(map[string]interface{})
for _, property := range types.UnsupportedProperties {
if _, isSet := serviceDict[property]; isSet {
unsupported[property] = true
}
}
}
}
Expand All @@ -168,8 +179,17 @@ func sortedKeys(set map[string]bool) []string {

// GetDeprecatedProperties returns the list of any deprecated properties that
// are used in the compose files.
func GetDeprecatedProperties(configDetails types.ConfigDetails) map[string]string {
return getProperties(getServices(getConfigDict(configDetails)), types.DeprecatedProperties)
func GetDeprecatedProperties(configDicts ...map[string]interface{}) map[string]string {
deprecated := map[string]string{}

for _, configDict := range configDicts {
deprecatedProperties := getProperties(getServices(configDict), types.DeprecatedProperties)
for key, value := range deprecatedProperties {
deprecated[key] = value
}
}

return deprecated
}

func getProperties(services map[string]interface{}, propertyMap map[string]string) map[string]string {
Expand Down Expand Up @@ -198,11 +218,6 @@ func (e *ForbiddenPropertiesError) Error() string {
return "Configuration contains forbidden properties"
}

// TODO: resolve multiple files into a single config
func getConfigDict(configDetails types.ConfigDetails) map[string]interface{} {
return configDetails.ConfigFiles[0].Config
}

func getServices(configDict map[string]interface{}) map[string]interface{} {
if services, ok := configDict["services"]; ok {
if servicesDict, ok := services.(map[string]interface{}); ok {
Expand Down
114 changes: 114 additions & 0 deletions cli/compose/loader/loader_multiple_test.go
@@ -0,0 +1,114 @@
package loader

import (
"testing"

"github.com/docker/cli/cli/compose/types"
"github.com/stretchr/testify/require"
)

/*
func TestLoadTwoDifferentVersion(t *testing.T) {
configDetails := types.ConfigDetails{
ConfigFiles: []types.ConfigFile{
{Filename: "base.yml", Config: map[string]interface{}{
"version": "2.0",
}},
{Filename: "override.yml", Config: map[string]interface{}{
"version": "3.0",
}},
},
}
_, err := Load(configDetails)
require.Error(t, err)
}
*/

func TestSimpleLoad(t *testing.T) {
base := map[string]interface{}{
"version": "3.4",
"services": map[string]interface{}{
"foo": map[string]interface{}{
"image": "foo",
"build": map[string]interface{}{
"context": ".",
"dockerfile": "bar.Dockerfile",
},
"labels": []interface{}{
"foo=bar",
},
"cap_add": []interface{}{
"NET_ADMIN",
},
},
},
"volumes": map[string]interface{}{},
"networks": map[string]interface{}{},
"secrets": map[string]interface{}{},
"configs": map[string]interface{}{},
}
override := map[string]interface{}{
"version": "3.4",
"services": map[string]interface{}{
"foo": map[string]interface{}{
"image": "baz",
"build": map[string]interface{}{
"dockerfile": "foo.Dockerfile",
"args": []interface{}{
"buildno=1",
"password=secret",
},
},
"labels": map[string]interface{}{
"foo": "baz",
},
"cap_add": []interface{}{
"SYS_ADMIN",
},
},
"bar": map[string]interface{}{
"image": "bar",
},
},
"volumes": map[string]interface{}{},
"networks": map[string]interface{}{},
"secrets": map[string]interface{}{},
"configs": map[string]interface{}{},
}
configDetails := types.ConfigDetails{
ConfigFiles: []types.ConfigFile{
{Filename: "base.yml", Config: base},
{Filename: "override.yml", Config: override},
},
}
config, err := Load(configDetails)
require.NoError(t, err)
require.Equal(t, &types.Config{
Services: []types.ServiceConfig{
{
Name: "bar",
Image: "bar",
Environment: types.MappingWithEquals{},
},
{
Name: "foo",
Image: "baz",
Build: types.BuildConfig{
Dockerfile: "foo.Dockerfile",
Args: types.MappingWithEquals{
"buildno": strPtr("1"),
"password": strPtr("secret"),
},
},
Labels: types.Labels{
"foo": "baz",
},
CapAdd: []string{"NET_ADMIN", "SYS_ADMIN"},
Environment: types.MappingWithEquals{},
}},
Networks: map[string]types.NetworkConfig{},
Volumes: map[string]types.VolumeConfig{},
Secrets: map[string]types.SecretConfig{},
Configs: map[string]types.ConfigObjConfig{},
}, config)
}
4 changes: 2 additions & 2 deletions cli/compose/loader/loader_test.go
Expand Up @@ -668,7 +668,7 @@ services:
_, err = Load(configDetails)
require.NoError(t, err)

unsupported := GetUnsupportedProperties(configDetails)
unsupported := GetUnsupportedProperties(dict)
assert.Equal(t, []string{"build", "links"}, unsupported)
}

Expand Down Expand Up @@ -711,7 +711,7 @@ services:
_, err = Load(configDetails)
require.NoError(t, err)

deprecated := GetDeprecatedProperties(configDetails)
deprecated := GetDeprecatedProperties(dict)
assert.Len(t, deprecated, 2)
assert.Contains(t, deprecated, "container_name")
assert.Contains(t, deprecated, "expose")
Expand Down

0 comments on commit 4809114

Please sign in to comment.