Permalink
Browse files

Config file handling improvements:

 * Load multiple configuration files from a directory
 * Ignore empty configuration files
 * Filter '#' style comments from JSON configs
  • Loading branch information...
mipearson committed Oct 19, 2014
1 parent e12e341 commit a1fe078042a33788d63d34d16d7ee94b23b1a18f
Showing with 299 additions and 110 deletions.
  1. +9 −7 README.md
  2. +101 −10 config.go
  3. +169 −86 config_test.go
  4. +20 −7 logstash-forwarder.go
View
@@ -37,9 +37,9 @@ logstash-forwarder is configured with a json file you specify with the -config f
`logstash-forwarder -config yourstuff.json`
Here's a sample, with comments in-line to describe the settings. Please please
please keep in mind that comments are technically invalid in JSON, so you can't
include them in your config.:
Here's a sample, with comments in-line to describe the settings. Comments are
invalid in JSON, but logstash-forwarder will strip them out for you if they're
the only thing on the line:
{
# The network section covers network configuration :)
@@ -71,7 +71,7 @@ include them in your config.:
# An array of hashes. Each hash tells what paths to watch and
# what fields to annotate on events from those paths.
{
"paths": [
"paths": [
# single paths are fine
"/var/log/messages",
# globs are fine too, they will be periodically evaluated
@@ -94,6 +94,8 @@ include them in your config.:
]
}
You can also read an entire directory of JSON configs by specifying a directory instead of a file with the `-config` option.
### Goals
* Minimize resource usage where possible (CPU, memory, network).
@@ -134,7 +136,7 @@ Or:
If you don't use rpm or deb make targets as above, you can skip this section.
Packages install to `/opt/logstash-forwarder`.
Packages install to `/opt/logstash-forwarder`.
There are no run-time dependencies.
@@ -190,7 +192,7 @@ In logstash, you'll want to use the [lumberjack](http://logstash.net/docs/latest
}
}
## Implementation details
## Implementation details
Below is valid as of 2012/09/19
@@ -236,7 +238,7 @@ reliable transport of events.
* HTTP is RPC and very high overhead for small events (uncompressable headers,
etc). Streaming requires custom framing.
## License
## License
See LICENSE file.
View
111 config.go
@@ -1,18 +1,23 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"regexp"
"time"
)
const configFileSizeLimit = 10 << 20
var defaultConfig = &struct {
netTimeout int64
fileDeadtime string
}{
netTimeout: 15,
netTimeout int64
fileDeadtime string
}{
netTimeout: 15,
fileDeadtime: "24h",
}
@@ -37,6 +42,61 @@ type FileConfig struct {
deadtime time.Duration
}
func DiscoverConfigs(file_or_directory string) (files []string, err error) {
fi, err := os.Stat(file_or_directory)
if err != nil {
return nil, err
}
files = make([]string, 0)
if fi.IsDir() {
entries, err := ioutil.ReadDir(file_or_directory)
if err != nil {
return nil, err
}
for _, filename := range entries {
files = append(files, path.Join(file_or_directory, filename.Name()))
}
} else {
files = append(files, file_or_directory)
}
return files, nil
}
// Append values to the 'to' config from the 'from' config, erroring
// if a value would be overwritten by the merge.
func MergeConfig(to *Config, from Config) (err error) {
to.Network.Servers = append(to.Network.Servers, from.Network.Servers...)
to.Files = append(to.Files, from.Files...)
// TODO: Is there a better way to do this in Go?
if from.Network.SSLCertificate != "" {
if to.Network.SSLCertificate != "" {
return fmt.Errorf("SSLCertificate already defined as '%s' in previous config file", to.Network.SSLCertificate)
}
to.Network.SSLCertificate = from.Network.SSLCertificate
}
if from.Network.SSLKey != "" {
if to.Network.SSLKey != "" {
return fmt.Errorf("SSLKey already defined as '%s' in previous config file", to.Network.SSLKey)
}
to.Network.SSLKey = from.Network.SSLKey
}
if from.Network.SSLCA != "" {
if to.Network.SSLCA != "" {
return fmt.Errorf("SSLCA already defined as '%s' in previous config file", to.Network.SSLCA)
}
to.Network.SSLCA = from.Network.SSLCA
}
if from.Network.Timeout != 0 {
if to.Network.Timeout != 0 {
return fmt.Errorf("Timeout already defined as '%d' in previous config file", to.Network.Timeout)
}
to.Network.Timeout = from.Network.Timeout
}
return nil
}
func LoadConfig(path string) (config Config, err error) {
config_file, err := os.Open(path)
if err != nil {
@@ -50,22 +110,27 @@ func LoadConfig(path string) (config Config, err error) {
return // REVU: shouldn't this return an error, then?
}
if fi.Size() == 0 {
emit("config file (%q) is empty, skipping", path)
return
}
buffer := make([]byte, fi.Size())
_, err = config_file.Read(buffer)
emit("%s\n", buffer)
err = json.Unmarshal(buffer, &config)
buffer, err = StripComments(buffer)
if err != nil {
emit("Failed unmarshalling json: %s\n", err)
emit("Failed to strip comments from json: %s\n", err)
return
}
if config.Network.Timeout == 0 {
config.Network.Timeout = defaultConfig.netTimeout
err = json.Unmarshal(buffer, &config)
if err != nil {
emit("Failed unmarshalling json: %s\n", err)
return
}
config.Network.timeout = time.Duration(config.Network.Timeout) * time.Second
for k, _ := range config.Files {
if config.Files[k].DeadTime == "" {
config.Files[k].DeadTime = defaultConfig.fileDeadtime
@@ -79,3 +144,29 @@ func LoadConfig(path string) (config Config, err error) {
return
}
func FinalizeConfig(config *Config) {
if config.Network.Timeout == 0 {
config.Network.Timeout = defaultConfig.netTimeout
}
config.Network.timeout = time.Duration(config.Network.Timeout) * time.Second
}
func StripComments(data []byte) ([]byte, error) {
data = bytes.Replace(data, []byte("\r"), []byte(""), 0) // Windows
lines := bytes.Split(data, []byte("\n"))
filtered := make([][]byte, 0)
for _, line := range lines {
match, err := regexp.Match(`^\s*#`, line)
if err != nil {
return nil, err
}
if !match {
filtered = append(filtered, line)
}
}
return bytes.Join(filtered, []byte("\n")), nil
}
Oops, something went wrong.

0 comments on commit a1fe078

Please sign in to comment.