Allow file provider to load config from files in a directory.

docs/toml.md

@@ -445,7 +445,7 @@ entryPoint = "https"
 
 ## File backend
 
-Like any other reverse proxy, Træfik can be configured with a file. You have two choices:
+Like any other reverse proxy, Træfik can be configured with a file. You have three choices:
 
 - simply add your configuration at the end of the global configuration file `traefik.toml`:
 
@@ -586,13 +586,21 @@ filename = "rules.toml"
     rule = "Path:/test"
 ```
 
+- or you could have multiple .toml files in a directory:
+
+```toml
+[file]
+directory = "/path/to/config/"
+```
+
 If you want Træfik to watch file changes automatically, just add:
 
 ```toml
 [file]
 watch = true
 ```
 
+
 ## API backend
 
 Træfik can be configured using a RESTful api.

integration/file_test.go

@@ -26,7 +26,7 @@ func (s *FileSuite) TestSimpleConfiguration(c *check.C) {
 	defer cmd.Process.Kill()
 
 	// Expected a 404 as we did not configure anything
-	try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
+	err = try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
 	c.Assert(err, checker.IsNil)
 }
 
@@ -38,6 +38,22 @@ func (s *FileSuite) TestSimpleConfigurationNoPanic(c *check.C) {
 	defer cmd.Process.Kill()
 
 	// Expected a 404 as we did not configure anything
-	try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
+	err = try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
+	c.Assert(err, checker.IsNil)
+}
+
+func (s *FileSuite) TestDirectoryConfiguration(c *check.C) {
+	cmd := exec.Command(traefikBinary, "--configFile=fixtures/file/directory.toml")
+
+	err := cmd.Start()
+	c.Assert(err, checker.IsNil)
+	defer cmd.Process.Kill()
+
+	// Expected a 404 as we did not configure anything at /test
+	err = try.GetRequest("http://127.0.0.1:8000/test", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
+	c.Assert(err, checker.IsNil)
+
+	// Expected a 502 as there is no backend server
+	err = try.GetRequest("http://127.0.0.1:8000/test2", 1000*time.Millisecond, try.StatusCodeIs(http.StatusBadGateway))
 	c.Assert(err, checker.IsNil)
 }

integration/fixtures/file/dir/simple1.toml

@@ -0,0 +1,11 @@
+# rules
+[backends]
+  [backends.backend1]
+    [backends.backend1.servers.server1]
+    url = "http://172.17.0.2:80"
+
+[frontends]
+  [frontends.frontend1]
+  backend = "backend1"
+    [frontends.frontend1.routes.test_1]
+    rule = "Path:/test1"

integration/fixtures/file/dir/simple2.toml

@@ -0,0 +1,11 @@
+# rules
+[backends]
+  [backends.backend2]
+    [backends.backend2.servers.server1]
+    url = "http://172.17.0.2:80"
+
+[frontends]
+  [frontends.frontend2]
+  backend = "backend2"
+    [frontends.frontend2.routes.test_2]
+    rule = "Path:/test2"

integration/fixtures/file/directory.toml

@@ -0,0 +1,10 @@
+defaultEntryPoints = ["http"]
+
+logLevel = "DEBUG"
+
+[entryPoints]
+  [entryPoints.http]
+  address = ":8000"
+
+[file]
+  directory = "fixtures/file/dir/"

provider/file/file.go

@@ -1,8 +1,9 @@
 package file
 
 import (
-	"os"
-	"path/filepath"
+	"fmt"
+	"io/ioutil"
+	"path"
 	"strings"
 
 	"github.com/BurntSushi/toml"
@@ -18,68 +19,138 @@ var _ provider.Provider = (*Provider)(nil)
 // Provider holds configurations of the provider.
 type Provider struct {
 	provider.BaseProvider `mapstructure:",squash"`
+	Directory             string `description:"Load configuration from one or more .toml files in a directory"`
 }
 
 // Provide allows the file provider to provide configurations to traefik
 // using the given configuration channel.
 func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
-	watcher, err := fsnotify.NewWatcher()
+	configuration, err := p.loadConfig()
+
 	if err != nil {
-		log.Error("Error creating file watcher", err)
 		return err
 	}
 
-	file, err := os.Open(p.Filename)
+	if p.Watch {
+		var watchItem string
+
+		if p.Directory != "" {
+			watchItem = p.Directory
+		} else {
+			watchItem = p.Filename
+		}
+
+		if err := p.addWatcher(pool, watchItem, configurationChan, p.watcherCallback); err != nil {
+			return err
+		}
+	}
+
+	sendConfigToChannel(configurationChan, configuration)
+	return nil
+}
+
+func (p *Provider) addWatcher(pool *safe.Pool, directory string, configurationChan chan<- types.ConfigMessage, callback func(chan<- types.ConfigMessage, fsnotify.Event)) error {
+	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
-		log.Error("Error opening file", err)
-		return err
+		return fmt.Errorf("error creating file watcher: %s", err)
 	}
-	defer file.Close()
 
-	if p.Watch {
-		// Process events
-		pool.Go(func(stop chan bool) {
-			defer watcher.Close()
-			for {
-				select {
-				case <-stop:
-					return
-				case event := <-watcher.Events:
-					if strings.Contains(event.Name, file.Name()) {
-						log.Debug("Provider event:", event)
-						configuration := p.loadFileConfig(file.Name())
-						if configuration != nil {
-							configurationChan <- types.ConfigMessage{
-								ProviderName:  "file",
-								Configuration: configuration,
-							}
-						}
-					}
-				case error := <-watcher.Errors:
-					log.Error("Watcher event error", error)
-				}
+	// Process events
+	pool.Go(func(stop chan bool) {
+		defer watcher.Close()
+		for {
+			select {
+			case <-stop:
+				return
+			case evt := <-watcher.Events:
+				callback(configurationChan, evt)
+			case err := <-watcher.Errors:
+				log.Errorf("Watcher event error: %s", err)
 			}
-		})
-		err = watcher.Add(filepath.Dir(file.Name()))
-		if err != nil {
-			log.Error("Error adding file watcher", err)
-			return err
 		}
+	})
+	err = watcher.Add(directory)
+	if err != nil {
+		return fmt.Errorf("error adding file watcher: %s", err)
 	}
 
-	configuration := p.loadFileConfig(file.Name())
+	return nil
+}
+
+func sendConfigToChannel(configurationChan chan<- types.ConfigMessage, configuration *types.Configuration) {
 	configurationChan <- types.ConfigMessage{
 		ProviderName:  "file",
 		Configuration: configuration,
 	}
-	return nil
 }
 
-func (p *Provider) loadFileConfig(filename string) *types.Configuration {
+func loadFileConfig(filename string) (*types.Configuration, error) {
 	configuration := new(types.Configuration)
 	if _, err := toml.DecodeFile(filename, configuration); err != nil {
-		log.Error("Error reading file:", err)
-		return nil
+		return nil, fmt.Errorf("error reading configuration file: %s", err)
+	}
+	return configuration, nil
+}
+
+func loadFileConfigFromDirectory(directory string) (*types.Configuration, error) {
+	fileList, err := ioutil.ReadDir(directory)
+
+	if err != nil {
+		return nil, fmt.Errorf("unable to read directory %s: %v", directory, err)
+	}
+
+	configuration := &types.Configuration{
+		Frontends: make(map[string]*types.Frontend),
+		Backends:  make(map[string]*types.Backend),
 	}
-	return configuration
+
+	for _, file := range fileList {
+		if !strings.HasSuffix(file.Name(), ".toml") {
+			continue
+		}
+
+		var c *types.Configuration
+		c, err = loadFileConfig(path.Join(directory, file.Name()))
+
+		if err != nil {
+			return nil, err
+		}
+
+		for backendName, backend := range c.Backends {
+			if _, exists := configuration.Backends[backendName]; exists {
+				log.Warnf("Backend %s already configured, skipping", backendName)
+			} else {
+				configuration.Backends[backendName] = backend
+			}
+		}
+
+		for frontendName, frontend := range c.Frontends {
+			if _, exists := configuration.Frontends[frontendName]; exists {
+				log.Warnf("Frontend %s already configured, skipping", frontendName)
+			} else {
+				configuration.Frontends[frontendName] = frontend
+			}
+		}
+	}
+
+	return configuration, nil
+}
+
+func (p *Provider) watcherCallback(configurationChan chan<- types.ConfigMessage, event fsnotify.Event) {
+	configuration, err := p.loadConfig()
+
+	if err != nil {
+		log.Errorf("Error occurred during watcher callback: %s", err)
+		return
+	}
+
+	sendConfigToChannel(configurationChan, configuration)
+}
+
+func (p *Provider) loadConfig() (*types.Configuration, error) {
+	if p.Directory != "" {
+		return loadFileConfigFromDirectory(p.Directory)
+	}
+
+	return loadFileConfig(p.Filename)
 }