Skip to content

Commit

Permalink
Add support for multiple composefile when deploying
Browse files Browse the repository at this point in the history
Using a custom version of mergo to manage special cases

Signed-off-by: Vincent Demeester <vincent@sbr.pm>
  • Loading branch information
vdemeester committed Sep 29, 2017
1 parent 10e292d commit 2455685
Show file tree
Hide file tree
Showing 14 changed files with 927 additions and 64 deletions.
6 changes: 3 additions & 3 deletions cli/command/stack/deploy.go
Expand Up @@ -22,7 +22,7 @@ const (

type deployOptions struct {
bundlefile string
composefile string
composefile []string
namespace string
resolveImage string
sendRegistryAuth bool
Expand Down Expand Up @@ -63,9 +63,9 @@ func runDeploy(dockerCli command.Cli, opts deployOptions) error {
}

switch {
case opts.bundlefile == "" && opts.composefile == "":
case opts.bundlefile == "" && len(opts.composefile) == 0:
return errors.Errorf("Please specify either a bundle file (with --bundle-file) or a Compose file (with --compose-file).")
case opts.bundlefile != "" && opts.composefile != "":
case opts.bundlefile != "" && len(opts.composefile) != 0:
return errors.Errorf("You cannot specify both a bundle file and a Compose file.")
case opts.bundlefile != "":
return deployBundle(ctx, dockerCli, opts)
Expand Down
46 changes: 37 additions & 9 deletions cli/command/stack/deploy_composefile.go
Expand Up @@ -38,13 +38,14 @@ func deployCompose(ctx context.Context, dockerCli command.Cli, opts deployOption
return err
}

unsupportedProperties := loader.GetUnsupportedProperties(configDetails)
dicts := getDictsFrom(configDetails.ConfigFiles)
unsupportedProperties := loader.GetUnsupportedProperties(dicts...)
if len(unsupportedProperties) > 0 {
fmt.Fprintf(dockerCli.Err(), "Ignoring unsupported options: %s\n\n",
strings.Join(unsupportedProperties, ", "))
}

deprecatedProperties := loader.GetDeprecatedProperties(configDetails)
deprecatedProperties := loader.GetDeprecatedProperties(dicts...)
if len(deprecatedProperties) > 0 {
fmt.Fprintf(dockerCli.Err(), "Ignoring deprecated options:\n\n%s\n\n",
propertyWarnings(deprecatedProperties))
Expand Down Expand Up @@ -96,6 +97,16 @@ func deployCompose(ctx context.Context, dockerCli command.Cli, opts deployOption
return deployServices(ctx, dockerCli, services, namespace, opts.sendRegistryAuth, opts.resolveImage)
}

func getDictsFrom(configFiles []composetypes.ConfigFile) []map[string]interface{} {
dicts := []map[string]interface{}{}

for _, configFile := range configFiles {
dicts = append(dicts, configFile.Config)
}

return dicts
}

func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
serviceNetworks := map[string]struct{}{}
for _, serviceConfig := range serviceConfigs {
Expand All @@ -119,29 +130,32 @@ func propertyWarnings(properties map[string]string) string {
return strings.Join(msgs, "\n\n")
}

func getConfigDetails(composefile string, stdin io.Reader) (composetypes.ConfigDetails, error) {
func getConfigDetails(composefile []string, stdin io.Reader) (composetypes.ConfigDetails, error) {
var details composetypes.ConfigDetails

if composefile == "-" {
if len(composefile) == 0 {
return details, errors.New("no composefile")
}

if composefile[0] == "-" {
workingDir, err := os.Getwd()
if err != nil {
return details, err
}
details.WorkingDir = workingDir
} else {
absPath, err := filepath.Abs(composefile)
absPath, err := filepath.Abs(composefile[0])
if err != nil {
return details, err
}
details.WorkingDir = filepath.Dir(absPath)
}

configFile, err := getConfigFile(composefile, stdin)
var err error
details.ConfigFiles, err = loadConfigFiles(composefile, stdin)
if err != nil {
return details, err
}
// TODO: support multiple files
details.ConfigFiles = []composetypes.ConfigFile{*configFile}
details.Environment, err = buildEnvironment(os.Environ())
return details, err
}
Expand All @@ -159,7 +173,21 @@ func buildEnvironment(env []string) (map[string]string, error) {
return result, nil
}

func getConfigFile(filename string, stdin io.Reader) (*composetypes.ConfigFile, error) {
func loadConfigFiles(filenames []string, stdin io.Reader) ([]composetypes.ConfigFile, error) {
var configFiles []composetypes.ConfigFile

for _, filename := range filenames {
configFile, err := loadConfigFile(filename, stdin)
if err != nil {
return configFiles, err
}
configFiles = append(configFiles, *configFile)
}

return configFiles, nil
}

func loadConfigFile(filename string, stdin io.Reader) (*composetypes.ConfigFile, error) {
var bytes []byte
var err error

Expand Down
4 changes: 2 additions & 2 deletions cli/command/stack/opts.go
Expand Up @@ -10,8 +10,8 @@ import (
"github.com/spf13/pflag"
)

func addComposefileFlag(opt *string, flags *pflag.FlagSet) {
flags.StringVarP(opt, "compose-file", "c", "", "Path to a Compose file")
func addComposefileFlag(opt *[]string, flags *pflag.FlagSet) {
flags.StringSliceVarP(opt, "compose-file", "c", []string{}, "Path to a Compose file")
flags.SetAnnotation("compose-file", "version", []string{"1.25"})
}

Expand Down
106 changes: 58 additions & 48 deletions cli/compose/loader/loader.go
Expand Up @@ -45,55 +45,59 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
if len(configDetails.ConfigFiles) < 1 {
return nil, errors.Errorf("No files specified")
}
if len(configDetails.ConfigFiles) > 1 {
return nil, errors.Errorf("Multiple files are not yet supported")
}

configDict := getConfigDict(configDetails)
configs := []*types.Config{}

if services, ok := configDict["services"]; ok {
if servicesDict, ok := services.(map[string]interface{}); ok {
forbidden := getProperties(servicesDict, types.ForbiddenProperties)
for _, file := range configDetails.ConfigFiles {
configDict := file.Config

if services, ok := configDict["services"]; ok {
if servicesDict, ok := services.(map[string]interface{}); ok {
forbidden := getProperties(servicesDict, types.ForbiddenProperties)

if len(forbidden) > 0 {
return nil, &ForbiddenPropertiesError{Properties: forbidden}
if len(forbidden) > 0 {
return nil, &ForbiddenPropertiesError{Properties: forbidden}
}
}
}
}

if err := schema.Validate(configDict, schema.Version(configDict)); err != nil {
return nil, err
}
if err := schema.Validate(configDict, schema.Version(configDict)); err != nil {
return nil, err
}

cfg := types.Config{}
cfg := types.Config{}

config, err := interpolateConfig(configDict, configDetails.LookupEnv)
if err != nil {
return nil, err
}
config, err := interpolateConfig(configDict, configDetails.LookupEnv)
if err != nil {
return nil, err
}

cfg.Services, err = LoadServices(config["services"], configDetails.WorkingDir, configDetails.LookupEnv)
if err != nil {
return nil, err
}
cfg.Services, err = LoadServices(config["services"], configDetails.WorkingDir, configDetails.LookupEnv)
if err != nil {
return nil, err
}

cfg.Networks, err = LoadNetworks(config["networks"])
if err != nil {
return nil, err
}
cfg.Networks, err = LoadNetworks(config["networks"])
if err != nil {
return nil, err
}

cfg.Volumes, err = LoadVolumes(config["volumes"])
if err != nil {
return nil, err
}
cfg.Volumes, err = LoadVolumes(config["volumes"])
if err != nil {
return nil, err
}

cfg.Secrets, err = LoadSecrets(config["secrets"], configDetails.WorkingDir)
if err != nil {
return nil, err
cfg.Secrets, err = LoadSecrets(config["secrets"], configDetails.WorkingDir)
if err != nil {
return nil, err
}

cfg.Configs, err = LoadConfigObjs(config["configs"], configDetails.WorkingDir)

configs = append(configs, &cfg)
}

cfg.Configs, err = LoadConfigObjs(config["configs"], configDetails.WorkingDir)
return &cfg, err
return merge(configs)
}

func interpolateConfig(configDict map[string]interface{}, lookupEnv template.Mapping) (map[string]map[string]interface{}, error) {
Expand All @@ -116,14 +120,16 @@ func interpolateConfig(configDict map[string]interface{}, lookupEnv template.Map

// GetUnsupportedProperties returns the list of any unsupported properties that are
// used in the Compose files.
func GetUnsupportedProperties(configDetails types.ConfigDetails) []string {
func GetUnsupportedProperties(configDicts ...map[string]interface{}) []string {
unsupported := map[string]bool{}

for _, service := range getServices(getConfigDict(configDetails)) {
serviceDict := service.(map[string]interface{})
for _, property := range types.UnsupportedProperties {
if _, isSet := serviceDict[property]; isSet {
unsupported[property] = true
for _, configDict := range configDicts {
for _, service := range getServices(configDict) {
serviceDict := service.(map[string]interface{})
for _, property := range types.UnsupportedProperties {
if _, isSet := serviceDict[property]; isSet {
unsupported[property] = true
}
}
}
}
Expand All @@ -142,8 +148,17 @@ func sortedKeys(set map[string]bool) []string {

// GetDeprecatedProperties returns the list of any deprecated properties that
// are used in the compose files.
func GetDeprecatedProperties(configDetails types.ConfigDetails) map[string]string {
return getProperties(getServices(getConfigDict(configDetails)), types.DeprecatedProperties)
func GetDeprecatedProperties(configDicts ...map[string]interface{}) map[string]string {
deprecated := map[string]string{}

for _, configDict := range configDicts {
deprecatedProperties := getProperties(getServices(configDict), types.DeprecatedProperties)
for key, value := range deprecatedProperties {
deprecated[key] = value
}
}

return deprecated
}

func getProperties(services map[string]interface{}, propertyMap map[string]string) map[string]string {
Expand Down Expand Up @@ -172,11 +187,6 @@ func (e *ForbiddenPropertiesError) Error() string {
return "Configuration contains forbidden properties"
}

// TODO: resolve multiple files into a single config
func getConfigDict(configDetails types.ConfigDetails) map[string]interface{} {
return configDetails.ConfigFiles[0].Config
}

func getServices(configDict map[string]interface{}) map[string]interface{} {
if services, ok := configDict["services"]; ok {
if servicesDict, ok := services.(map[string]interface{}); ok {
Expand Down
114 changes: 114 additions & 0 deletions cli/compose/loader/loader_multiple_test.go
@@ -0,0 +1,114 @@
package loader

import (
"testing"

"github.com/docker/cli/cli/compose/types"
"github.com/stretchr/testify/require"
)

/*
func TestLoadTwoDifferentVersion(t *testing.T) {
configDetails := types.ConfigDetails{
ConfigFiles: []types.ConfigFile{
{Filename: "base.yml", Config: map[string]interface{}{
"version": "2.0",
}},
{Filename: "override.yml", Config: map[string]interface{}{
"version": "3.0",
}},
},
}
_, err := Load(configDetails)
require.Error(t, err)
}
*/

func TestSimpleLoad(t *testing.T) {
base := map[string]interface{}{
"version": "3.4",
"services": map[string]interface{}{
"foo": map[string]interface{}{
"image": "foo",
"build": map[string]interface{}{
"context": ".",
"dockerfile": "bar.Dockerfile",
},
"labels": []interface{}{
"foo=bar",
},
"cap_add": []interface{}{
"NET_ADMIN",
},
},
},
"volumes": map[string]interface{}{},
"networks": map[string]interface{}{},
"secrets": map[string]interface{}{},
"configs": map[string]interface{}{},
}
override := map[string]interface{}{
"version": "3.4",
"services": map[string]interface{}{
"foo": map[string]interface{}{
"image": "baz",
"build": map[string]interface{}{
"dockerfile": "foo.Dockerfile",
"args": []interface{}{
"buildno=1",
"password=secret",
},
},
"labels": map[string]interface{}{
"foo": "baz",
},
"cap_add": []interface{}{
"SYS_ADMIN",
},
},
"bar": map[string]interface{}{
"image": "bar",
},
},
"volumes": map[string]interface{}{},
"networks": map[string]interface{}{},
"secrets": map[string]interface{}{},
"configs": map[string]interface{}{},
}
configDetails := types.ConfigDetails{
ConfigFiles: []types.ConfigFile{
{Filename: "base.yml", Config: base},
{Filename: "override.yml", Config: override},
},
}
config, err := Load(configDetails)
require.NoError(t, err)
require.Equal(t, &types.Config{
Services: []types.ServiceConfig{
{
Name: "bar",
Image: "bar",
Environment: types.MappingWithEquals{},
},
{
Name: "foo",
Image: "baz",
Build: types.BuildConfig{
Dockerfile: "foo.Dockerfile",
Args: types.MappingWithEquals{
"buildno": strPtr("1"),
"password": strPtr("secret"),
},
},
Labels: types.Labels{
"foo": "baz",
},
CapAdd: []string{"NET_ADMIN", "SYS_ADMIN"},
Environment: types.MappingWithEquals{},
}},
Networks: map[string]types.NetworkConfig{},
Volumes: map[string]types.VolumeConfig{},
Secrets: map[string]types.SecretConfig{},
Configs: map[string]types.ConfigObjConfig{},
}, config)
}

0 comments on commit 2455685

Please sign in to comment.