diff --git a/go.mod b/go.mod index f69d496..2d46709 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 github.com/dennwc/btrfs v0.0.0-20190517175702-d917b30ff035 github.com/gin-gonic/gin v1.5.0 + github.com/imdario/mergo v0.3.9 github.com/mattn/goveralls v0.0.5 // indirect github.com/pkg/profile v1.4.0 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 diff --git a/go.sum b/go.sum index 8e28227..f76dd97 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,9 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= +github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -38,7 +41,9 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/goveralls v0.0.5 h1:spfq8AyZ0cCk57Za6/juJ5btQxeE1FaEGMdfcI+XO48= github.com/mattn/goveralls v0.0.5/go.mod h1:Xg2LHi51faXLyKXwsndxiW6uxEEQT9+3sjGzzwU4xy0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI= github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE= diff --git a/worker/config.go b/worker/config.go index 34b2599..2f391b3 100644 --- a/worker/config.go +++ b/worker/config.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/BurntSushi/toml" + "github.com/imdario/mergo" ) type providerEnum uint8 @@ -41,7 +42,8 @@ type Config struct { BtrfsSnapshot btrfsSnapshotConfig `toml:"btrfs_snapshot"` Docker dockerConfig `toml:"docker"` Include includeConfig `toml:"include"` - Mirrors []mirrorConfig `toml:"mirrors"` + MirrorsConf []mirrorConfig `toml:"mirrors"` + Mirrors []mirrorConfig } type globalConfig struct { @@ -117,6 +119,7 @@ type mirrorConfig struct { Interval int `toml:"interval"` Retry int `toml:"retry"` MirrorDir string `toml:"mirror_dir"` + MirrorSubDir string `toml:"mirror_subdir"` LogDir string `toml:"log_dir"` Env map[string]string `toml:"env"` Role string `toml:"role"` @@ -148,6 +151,8 @@ type mirrorConfig struct { DockerOptions []string `toml:"docker_options"` SnapshotPath string `toml:"snapshot_path"` + + ChildMirrors []mirrorConfig `toml:"mirrors"` } // LoadConfig loads configuration @@ -174,9 +179,36 @@ func LoadConfig(cfgFile string) (*Config, error) { logger.Errorf(err.Error()) return nil, err } - cfg.Mirrors = append(cfg.Mirrors, incMirCfg.Mirrors...) + cfg.MirrorsConf = append(cfg.MirrorsConf, incMirCfg.Mirrors...) + } + } + + for _, m := range cfg.MirrorsConf { + if err := recursiveMirrors(cfg, nil, m); err != nil { + return nil, err; } } return cfg, nil } + +func recursiveMirrors(cfg *Config, parent *mirrorConfig, mirror mirrorConfig) error { + var curMir mirrorConfig; + if parent != nil { + curMir = *parent; + } + curMir.ChildMirrors = nil; + if err := mergo.Merge(&curMir, mirror, mergo.WithOverride); err != nil { + return err; + } + if mirror.ChildMirrors == nil { + cfg.Mirrors = append(cfg.Mirrors, curMir); + } else { + for _, m := range mirror.ChildMirrors { + if err := recursiveMirrors(cfg, &curMir, m); err != nil { + return err; + } + } + } + return nil; +} diff --git a/worker/config_test.go b/worker/config_test.go index b35c99d..4114097 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -83,9 +83,9 @@ exec_on_failure = [ tmpDir, ) - cfgBlob = cfgBlob + incSection + curCfgBlob := cfgBlob + incSection - err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644) + err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644) So(err, ShouldEqual, nil) defer tmpfile.Close() @@ -157,6 +157,102 @@ use_ipv6 = true So(len(cfg.Mirrors), ShouldEqual, 6) }) + Convey("Everything should work on nested config file", t, func() { + tmpfile, err := ioutil.TempFile("", "tunasync") + So(err, ShouldEqual, nil) + defer os.Remove(tmpfile.Name()) + + tmpDir, err := ioutil.TempDir("", "tunasync") + So(err, ShouldBeNil) + defer os.RemoveAll(tmpDir) + + incSection := fmt.Sprintf( + "\n[include]\n"+ + "include_mirrors = \"%s/*.conf\"", + tmpDir, + ) + + curCfgBlob := cfgBlob + incSection + + err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644) + So(err, ShouldEqual, nil) + defer tmpfile.Close() + + incBlob1 := ` +[[mirrors]] +name = "ipv6s" +use_ipv6 = true + [[mirrors.mirrors]] + name = "debians" + mirror_subdir = "debian" + provider = "two-stage-rsync" + stage1_profile = "debian" + + [[mirrors.mirrors.mirrors]] + name = "debian-security" + upstream = "rsync://test.host/debian-security/" + [[mirrors.mirrors.mirrors]] + name = "ubuntu" + stage1_profile = "ubuntu" + upstream = "rsync://test.host2/ubuntu/" + [[mirrors.mirrors]] + name = "debian-cd" + provider = "rsync" + upstream = "rsync://test.host3/debian-cd/" + ` + err = ioutil.WriteFile(filepath.Join(tmpDir, "nest.conf"), []byte(incBlob1), 0644) + So(err, ShouldEqual, nil) + + cfg, err := LoadConfig(tmpfile.Name()) + So(err, ShouldBeNil) + So(cfg.Global.Name, ShouldEqual, "test_worker") + So(cfg.Global.Interval, ShouldEqual, 240) + So(cfg.Global.Retry, ShouldEqual, 3) + So(cfg.Global.MirrorDir, ShouldEqual, "/data/mirrors") + + So(cfg.Manager.APIBase, ShouldEqual, "https://127.0.0.1:5000") + So(cfg.Server.Hostname, ShouldEqual, "worker1.example.com") + + m := cfg.Mirrors[0] + So(m.Name, ShouldEqual, "AOSP") + So(m.MirrorDir, ShouldEqual, "/data/git/AOSP") + So(m.Provider, ShouldEqual, provCommand) + So(m.Interval, ShouldEqual, 720) + So(m.Retry, ShouldEqual, 2) + So(m.Env["REPO"], ShouldEqual, "/usr/local/bin/aosp-repo") + + m = cfg.Mirrors[1] + So(m.Name, ShouldEqual, "debian") + So(m.MirrorDir, ShouldEqual, "") + So(m.Provider, ShouldEqual, provTwoStageRsync) + + m = cfg.Mirrors[2] + So(m.Name, ShouldEqual, "fedora") + So(m.MirrorDir, ShouldEqual, "") + So(m.Provider, ShouldEqual, provRsync) + So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt") + + m = cfg.Mirrors[3] + So(m.Name, ShouldEqual, "debian-security") + So(m.MirrorDir, ShouldEqual, "") + So(m.Provider, ShouldEqual, provTwoStageRsync) + So(m.UseIPv6, ShouldEqual, true) + So(m.Stage1Profile, ShouldEqual, "debian") + + m = cfg.Mirrors[4] + So(m.Name, ShouldEqual, "ubuntu") + So(m.MirrorDir, ShouldEqual, "") + So(m.Provider, ShouldEqual, provTwoStageRsync) + So(m.UseIPv6, ShouldEqual, true) + So(m.Stage1Profile, ShouldEqual, "ubuntu") + + m = cfg.Mirrors[5] + So(m.Name, ShouldEqual, "debian-cd") + So(m.UseIPv6, ShouldEqual, true) + So(m.Provider, ShouldEqual, provRsync) + + So(len(cfg.Mirrors), ShouldEqual, 6) + }) Convey("Providers can be inited from a valid config file", t, func() { tmpfile, err := ioutil.TempFile("", "tunasync") So(err, ShouldEqual, nil) @@ -207,4 +303,90 @@ use_ipv6 = true So(rp.excludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt") }) + + Convey("MirrorSubdir should work", t, func() { + tmpfile, err := ioutil.TempFile("", "tunasync") + So(err, ShouldEqual, nil) + defer os.Remove(tmpfile.Name()) + + cfgBlob1 := ` +[global] +name = "test_worker" +log_dir = "/var/log/tunasync/{{.Name}}" +mirror_dir = "/data/mirrors" +concurrent = 10 +interval = 240 +retry = 3 + +[manager] +api_base = "https://127.0.0.1:5000" +token = "some_token" + +[server] +hostname = "worker1.example.com" +listen_addr = "127.0.0.1" +listen_port = 6000 +ssl_cert = "/etc/tunasync.d/worker1.cert" +ssl_key = "/etc/tunasync.d/worker1.key" + +[[mirrors]] +name = "ipv6s" +use_ipv6 = true + [[mirrors.mirrors]] + name = "debians" + mirror_subdir = "debian" + provider = "two-stage-rsync" + stage1_profile = "debian" + + [[mirrors.mirrors.mirrors]] + name = "debian-security" + upstream = "rsync://test.host/debian-security/" + [[mirrors.mirrors.mirrors]] + name = "ubuntu" + stage1_profile = "ubuntu" + upstream = "rsync://test.host2/ubuntu/" + [[mirrors.mirrors]] + name = "debian-cd" + provider = "rsync" + upstream = "rsync://test.host3/debian-cd/" + ` + err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644) + So(err, ShouldEqual, nil) + defer tmpfile.Close() + + cfg, err := LoadConfig(tmpfile.Name()) + So(err, ShouldBeNil) + + providers := map[string]mirrorProvider{} + for _, m := range cfg.Mirrors { + p := newMirrorProvider(m, cfg) + providers[p.Name()] = p + } + + p := providers["debian-security"] + So(p.Name(), ShouldEqual, "debian-security") + So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian-security") + So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian-security/latest.log") + r2p, ok := p.(*twoStageRsyncProvider) + So(ok, ShouldBeTrue) + So(r2p.stage1Profile, ShouldEqual, "debian") + So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian/debian-security") + + p = providers["ubuntu"] + So(p.Name(), ShouldEqual, "ubuntu") + So(p.LogDir(), ShouldEqual, "/var/log/tunasync/ubuntu") + So(p.LogFile(), ShouldEqual, "/var/log/tunasync/ubuntu/latest.log") + r2p, ok = p.(*twoStageRsyncProvider) + So(ok, ShouldBeTrue) + So(r2p.stage1Profile, ShouldEqual, "ubuntu") + So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian/ubuntu") + + p = providers["debian-cd"] + So(p.Name(), ShouldEqual, "debian-cd") + So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian-cd") + So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian-cd/latest.log") + rp, ok := p.(*rsyncProvider) + So(ok, ShouldBeTrue) + So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/debian-cd") + }) } diff --git a/worker/provider.go b/worker/provider.go index 253f485..af05e02 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -82,7 +82,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider { } if mirrorDir == "" { mirrorDir = filepath.Join( - cfg.Global.MirrorDir, mirror.Name, + cfg.Global.MirrorDir, mirror.MirrorSubDir, mirror.Name, ) } if mirror.Interval == 0 {