Skip to content

Commit

Permalink
Revamped config package (#103)
Browse files Browse the repository at this point in the history 
* Revamped configutils

* Added --secrets flag

* Addressed comments

* Handle no secrets file
  • Loading branch information
@apourchet
apourchet committed Mar 9, 2019
1 parent 605a793 commit d86f045
Show file tree
Hide file tree
Showing 7 changed files with 116 additions and 308 deletions.
8 changes: 8 additions & 0 deletions agent/cmd/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ var (
configFile string
zone string
krakenCluster string
secretsFile string

rootCmd = &cobra.Command{
Short: "kraken-agent implements docker registry interface and downloads data as a peer " +
Expand All @@ -69,6 +70,8 @@ func init() {
&zone, "zone", "", "", "zone/datacenter name")
rootCmd.PersistentFlags().StringVarP(
&krakenCluster, "cluster", "", "", "cluster name (e.g. prod01-zone1)")
rootCmd.PersistentFlags().StringVarP(
&secretsFile, "secrets", "", "", "path to a secrets YAML file to load into configuration")
}

func Execute() {
Expand All @@ -89,6 +92,11 @@ func run() {
if err := configutil.Load(configFile, &config); err != nil {
panic(err)
}
if secretsFile != "" {
if err := configutil.Load(secretsFile, &config); err != nil {
panic(err)
}
}

zlog := log.ConfigureLogger(config.ZapLogging)
defer zlog.Sync()
Expand Down
8 changes: 8 additions & 0 deletions build-index/cmd/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ var (
port int
configFile string
krakenCluster string
secretsFile string

rootCmd = &cobra.Command{
Short: "kraken-index handles all tag related requests and cross cluster replications",
Expand All @@ -56,6 +57,8 @@ func init() {
&configFile, "config", "", "", "configuration file path")
rootCmd.PersistentFlags().StringVarP(
&krakenCluster, "cluster", "", "", "cluster name (e.g. prod01-zone1)")
rootCmd.PersistentFlags().StringVarP(
&secretsFile, "secrets", "", "", "path to a secrets YAML file to load into configuration")
}

func Execute() {
Expand All @@ -71,6 +74,11 @@ func run() {
if err := configutil.Load(configFile, &config); err != nil {
panic(err)
}
if secretsFile != "" {
if err := configutil.Load(secretsFile, &config); err != nil {
panic(err)
}
}
log.ConfigureLogger(config.ZapLogging)

stats, closer, err := metrics.New(config.Metrics, krakenCluster)
Expand Down
8 changes: 8 additions & 0 deletions origin/cmd/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ var (
configFile string
zone string
krakenCluster string
secretsFile string

rootCmd = &cobra.Command{
Short: "kraken-origin serves as dedicated seeder in kraken's p2p network.",
Expand All @@ -78,6 +79,8 @@ func init() {
&zone, "zone", "", "", "zone/datacenter name")
rootCmd.PersistentFlags().StringVarP(
&krakenCluster, "cluster", "", "", "cluster name (e.g. prod01-zone1)")
rootCmd.PersistentFlags().StringVarP(
&secretsFile, "secrets", "", "", "path to a secrets YAML file to load into configuration")
}

func Execute() {
Expand Down Expand Up @@ -108,6 +111,11 @@ func run() {
if err := configutil.Load(configFile, &config); err != nil {
panic(err)
}
if secretsFile != "" {
if err := configutil.Load(secretsFile, &config); err != nil {
panic(err)
}
}

zlog := log.ConfigureLogger(config.ZapLogging)
defer zlog.Sync()
Expand Down
8 changes: 8 additions & 0 deletions proxy/cmd/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ var (
ports []int
configFile string
krakenCluster string
secretsFile string

rootCmd = &cobra.Command{
Short: "kraken-proxy handles uploads and direct downloads",
Expand All @@ -49,6 +50,8 @@ func init() {
&configFile, "config", "", "", "configuration file path")
rootCmd.PersistentFlags().StringVarP(
&krakenCluster, "cluster", "", "", "cluster name (e.g. prod01-zone1)")
rootCmd.PersistentFlags().StringVarP(
&secretsFile, "secrets", "", "", "path to a secrets YAML file to load into configuration")
}

func Execute() {
Expand All @@ -64,6 +67,11 @@ func run() {
if err := configutil.Load(configFile, &config); err != nil {
panic(err)
}
if secretsFile != "" {
if err := configutil.Load(secretsFile, &config); err != nil {
panic(err)
}
}

log.ConfigureLogger(config.ZapLogging)

Expand Down
8 changes: 8 additions & 0 deletions tracker/cmd/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ var (
port int
configFile string
krakenCluster string
secretsFile string

rootCmd = &cobra.Command{
Short: "kraken-tracker keeps track of all the peers and their data in the p2p network.",
Expand All @@ -49,6 +50,8 @@ func init() {
&configFile, "config", "", "", "configuration file path")
rootCmd.PersistentFlags().StringVarP(
&krakenCluster, "cluster", "", "", "cluster name (e.g. prod01-zone1)")
rootCmd.PersistentFlags().StringVarP(
&secretsFile, "secrets", "", "", "path to a secrets YAML file to load into configuration")
}

func Execute() {
Expand All @@ -60,6 +63,11 @@ func run() {
if err := configutil.Load(configFile, &config); err != nil {
panic(err)
}
if secretsFile != "" {
if err := configutil.Load(secretsFile, &config); err != nil {
panic(err)
}
}
log.ConfigureLogger(config.ZapLogging)

stats, closer, err := metrics.New(config.Metrics, krakenCluster)
Expand Down
149 changes: 43 additions & 106 deletions utils/configutil/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,11 @@
// extends: base.yaml
//
// There is no multiple inheritance supported. Dependency tree suppossed to
// form a directed acyclic graph(DAG).
// form a linked list.
//
//
// Values from multiple configurations within the same hierarchy are deep merged
//
// Package support multiple configuraton directories potentially having multiple files
// with the the same name. In this the we just follow the path in extends and load all
// the file accroding to the relative directory, i.e configA: base.yaml production.yaml(extends base.yaml), configB: base.yaml the load sequance
// will be the following: configA(base.yaml), configA(production.yaml)
//
// Note regarding configuration merging:
// Array defined in YAML will be overriden based on load sequence.
// e.g. in the base.yaml:
Expand Down Expand Up @@ -63,29 +58,15 @@ import (
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"

"github.com/uber/kraken/utils/stringset"

"gopkg.in/validator.v2"
"gopkg.in/yaml.v2"
)

const (
configDirKey = "UBER_CONFIG_DIR"

configDir = "config"
secretsFile = "secrets.yaml"
configSeparator = ":"
)

// ErrNoFilesToLoad is returned when you attemp to call LoadFiles without valid
// file paths.
var ErrNoFilesToLoad = errors.New("attempt to load configuration with no files")

// ErrCycleRef is returned when there are circular dependencies detected in
// configuraiton files extending each other.
var ErrCycleRef = errors.New("cyclic reference in configuration extends detected")
Expand All @@ -106,6 +87,7 @@ func (e ValidationError) ErrForField(name string) error {
return e.errorMap[name]
}

// Error implements the `error` interface.
func (e ValidationError) Error() string {
var w bytes.Buffer

Expand All @@ -117,111 +99,66 @@ func (e ValidationError) Error() string {
return w.String()
}

// FilterCandidates filters candidate config files into only the ones that
// exist.
func FilterCandidates(fname string) ([]string, error) {
realConfigDirs := []string{configDir}
// Allow overriding the directory config is loaded from, useful for tests
// inside subdirectories when the config dir is at the top-level of a
// project.
if configRoot := os.Getenv(configDirKey); configRoot != "" {
realConfigDirs = strings.Split(configRoot, configSeparator)
}
return filterCandidatesFromDirs(fname, realConfigDirs)
}

func readExtend(configFile string) (string, error) {
var cfg Extends

data, err := ioutil.ReadFile(configFile)
// Load loads configuration based on config file name. It will
// follow extends directives and do a deep merge of those config
// files.
func Load(filename string, config interface{}) error {
filenames, err := resolveExtends(filename, readExtend)
if err != nil {
return "", err
}

if err := yaml.Unmarshal(data, &cfg); err != nil {
return "", fmt.Errorf("unmarshal %s: %s", configFile, err)
}
return cfg.Extends, nil
}

func getCandidate(fname string, dirs []string) string {
candidate := ""
for _, realConfigDir := range dirs {
configFile := path.Join(realConfigDir, fname)
if _, err := os.Stat(configFile); err == nil {
candidate = configFile
}
return err
}
return candidate
return loadFiles(config, filenames)
}

func filterCandidatesFromDirs(fname string, dirs []string) ([]string, error) {
var paths []string
cSet := make(stringset.Set)

// Go through all the 'extends' hierarchy until there is no base anymore or
// some reference cycles have been detected.
cSet.Add(fname)

candidate := getCandidate(fname, dirs)
fmt.Fprintf(os.Stderr, "candidate: %s\n", candidate)
if candidate == "" {
return nil, ErrNoFilesToLoad
}
type getExtend func(filename string) (extends string, err error)

// resolveExtends returns the list of config paths that the original config `filename`
// points to.
func resolveExtends(filename string, extendReader getExtend) ([]string, error) {
filenames := []string{filename}
seen := make(stringset.Set)
for {
extends, err := readExtend(candidate)
extends, err := extendReader(filename)
if err != nil {
return nil, err
} else if extends == "" {
break
}

paths = append([]string{candidate}, paths...)
if extends != "" {
// Prevent circular references.
if !cSet.Has(extends) {
candidate = path.Join(filepath.Dir(candidate), extends)
cSet.Add(extends)
} else {
return nil, ErrCycleRef
}
} else {
break
// If the file path of the extends field in the config is not absolute
// we assume that it is in the same directory as the current config
// file.
if !filepath.IsAbs(extends) {
extends = path.Join(filepath.Dir(filename), extends)
}
}

// Append secrets.
candidate = getCandidate(secretsFile, dirs)
if candidate != "" {
paths = append(paths, candidate)
}
// Prevent circular references.
if seen.Has(extends) {
return nil, ErrCycleRef
}

return paths, nil
filenames = append([]string{extends}, filenames...)
seen.Add(extends)
filename = extends
}
return filenames, nil
}

// Load loads configuration based on config file name.
// If config directory cannot be derived from file name, get it from environment
// variables.
func Load(fname string, config interface{}) error {
candidates, err := filterCandidatesFromDirs(
filepath.Base(fname), []string{filepath.Dir(fname)})
if err != nil && err != ErrNoFilesToLoad {
return fmt.Errorf("find config under %s: %s", filepath.Dir(fname), err)
} else if err == ErrNoFilesToLoad {
candidates, err = FilterCandidates(fname)
if err != nil {
return fmt.Errorf(
"find config under %s and %s: %s", filepath.Dir(fname), configDirKey, err)
}
func readExtend(configFile string) (string, error) {
data, err := ioutil.ReadFile(configFile)
if err != nil {
return "", err
}

return loadFiles(config, candidates...)
var cfg Extends
if err := yaml.Unmarshal(data, &cfg); err != nil {
return "", fmt.Errorf("unmarshal %s: %s", configFile, err)
}
return cfg.Extends, nil
}

// LoadFiles loads a list of files, deep-merging values.
func loadFiles(config interface{}, fnames ...string) error {
if len(fnames) == 0 {
return ErrNoFilesToLoad
}
// loadFiles loads a list of files, deep-merging values.
func loadFiles(config interface{}, fnames []string) error {
for _, fname := range fnames {
data, err := ioutil.ReadFile(fname)
if err != nil {
Expand Down
Loading

0 comments on commit d86f045

Please sign in to comment.