Skip to content
Permalink
Browse files

Merge pull request #1297 from weaveworks/issue/1267-multiple-paths

Support multiple `-git-path` arguments
  • Loading branch information
squaremo committed Aug 22, 2018
2 parents 74f0b7d + 8f09ae6 commit 670fce08d709d7154d64267982d8331ffef31eae
@@ -10,8 +10,8 @@ import (
type Manifests struct {
}

func (c *Manifests) LoadManifests(base, first string, rest ...string) (map[string]resource.Resource, error) {
return kresource.Load(base, first, rest...)
func (c *Manifests) LoadManifests(base string, paths []string) (map[string]resource.Resource, error) {
return kresource.Load(base, paths)
}

func (c *Manifests) ParseManifests(allDefs []byte) (map[string]resource.Resource, error) {
@@ -15,14 +15,13 @@ import (
// Load takes paths to directories or files, and creates an object set
// based on the file(s) therein. Resources are named according to the
// file content, rather than the file name of directory structure.
func Load(base, atLeastOne string, more ...string) (map[string]resource.Resource, error) {
roots := append([]string{atLeastOne}, more...)
func Load(base string, paths []string) (map[string]resource.Resource, error) {
objs := map[string]resource.Resource{}
charts, err := newChartTracker(base)
if err != nil {
return nil, errors.Wrapf(err, "walking %q for chartdirs", base)
}
for _, root := range roots {
for _, root := range paths {
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return errors.Wrapf(err, "walking %q for yamels", path)
@@ -200,7 +200,7 @@ func TestLoadSome(t *testing.T) {
if err := testfiles.WriteTestFiles(dir); err != nil {
t.Fatal(err)
}
objs, err := Load(dir, dir)
objs, err := Load(dir, []string{dir})
if err != nil {
t.Error(err)
}
@@ -231,7 +231,7 @@ func TestChartTracker(t *testing.T) {
if f == "garbage" {
continue
}
if m, err := Load(dir, fq); err != nil || len(m) == 0 {
if m, err := Load(dir, []string{fq}); err != nil || len(m) == 0 {
t.Errorf("Load returned 0 objs, err=%v", err)
}
}
@@ -250,7 +250,7 @@ func TestChartTracker(t *testing.T) {
}
for _, f := range chartfiles {
fq := filepath.Join(dir, f)
if m, err := Load(dir, fq); err != nil || len(m) != 0 {
if m, err := Load(dir, []string{fq}); err != nil || len(m) != 0 {
t.Errorf("%q not ignored as a chart should be", f)
}
}
@@ -26,11 +26,11 @@ func ErrResourceNotFound(name string) error {
type Manifests interface {
// Update the image in a manifest's bytes to that given
UpdateImage(def []byte, resourceID flux.ResourceID, container string, newImageID image.Ref) ([]byte, error)
// Load all the resource manifests under the path given. `baseDir`
// is used to relativise the paths, which are supplied as absolute
// paths to directories or files; at least one path must be
// supplied.
LoadManifests(baseDir, first string, rest ...string) (map[string]resource.Resource, error)
// Load all the resource manifests under the paths
// given. `baseDir` is used to relativise the paths, which are
// supplied as absolute paths to directories or files; at least
// one path should be supplied, even if it is the same as `baseDir`.
LoadManifests(baseDir string, paths []string) (map[string]resource.Resource, error)
// Parse the manifests given in an exported blob
ParseManifests([]byte) (map[string]resource.Resource, error)
// UpdatePolicies modifies a manifest to apply the policy update specified
@@ -40,8 +40,8 @@ type Manifests interface {
// UpdateManifest looks for the manifest for the identified resource,
// reads its contents, applies f(contents), and writes the results
// back to the file.
func UpdateManifest(m Manifests, root string, id flux.ResourceID, f func(manifest []byte) ([]byte, error)) error {
resources, err := m.LoadManifests(root, root)
func UpdateManifest(m Manifests, root string, paths []string, id flux.ResourceID, f func(manifest []byte) ([]byte, error)) error {
resources, err := m.LoadManifests(root, paths)
if err != nil {
return err
}
@@ -17,7 +17,7 @@ type Mock struct {
SyncFunc func(SyncDef) error
PublicSSHKeyFunc func(regenerate bool) (ssh.PublicKey, error)
UpdateImageFunc func(def []byte, id flux.ResourceID, container string, newImageID image.Ref) ([]byte, error)
LoadManifestsFunc func(base, first string, rest ...string) (map[string]resource.Resource, error)
LoadManifestsFunc func(base string, paths []string) (map[string]resource.Resource, error)
ParseManifestsFunc func([]byte) (map[string]resource.Resource, error)
UpdateManifestFunc func(path, resourceID string, f func(def []byte) ([]byte, error)) error
UpdatePoliciesFunc func([]byte, flux.ResourceID, policy.Update) ([]byte, error)
@@ -51,8 +51,8 @@ func (m *Mock) UpdateImage(def []byte, id flux.ResourceID, container string, new
return m.UpdateImageFunc(def, id, container, newImageID)
}

func (m *Mock) LoadManifests(base, first string, rest ...string) (map[string]resource.Resource, error) {
return m.LoadManifestsFunc(base, first, rest...)
func (m *Mock) LoadManifests(base string, paths []string) (map[string]resource.Resource, error) {
return m.LoadManifestsFunc(base, paths)
}

func (m *Mock) ParseManifests(def []byte) (map[string]resource.Resource, error) {
@@ -79,7 +79,7 @@ func main() {
// Git repo & key etc.
gitURL = fs.String("git-url", "", "URL of git repo with Kubernetes manifests; e.g., git@github.com:weaveworks/flux-example")
gitBranch = fs.String("git-branch", "master", "branch of git repo to use for Kubernetes manifests")
gitPath = fs.String("git-path", "", "path within git repo to locate Kubernetes manifests (relative path)")
gitPath = fs.StringSlice("git-path", []string{}, "relative paths within the git repo to locate Kubernetes manifests")
gitUser = fs.String("git-user", "Weave Flux", "username to use as git committer")
gitEmail = fs.String("git-email", "support@weave.works", "email to use as git committer")
gitSetAuthor = fs.Bool("git-set-author", false, "If set, the author of git commits will reflect the user who initiated the commit and will differ from the git committer.")
@@ -161,9 +161,11 @@ func main() {
*gitSkipMessage = defaultGitSkipMessage
}

if len(*gitPath) > 0 && (*gitPath)[0] == '/' {
logger.Log("err", "git subdirectory (--git-path) should not have leading forward slash")
os.Exit(1)
for _, path := range *gitPath {
if len(path) > 0 && path[0] == '/' {
logger.Log("err", "subdirectory given as --git-path should not have leading forward slash")
os.Exit(1)
}
}

if *sshKeygenDir == "" {
@@ -353,7 +355,7 @@ func main() {

gitRemote := git.Remote{URL: *gitURL}
gitConfig := git.Config{
Path: *gitPath,
Paths: *gitPath,
Branch: *gitBranch,
SyncTag: *gitSyncTag,
NotesRef: *gitNotesRef,
@@ -5,6 +5,7 @@ import (
"context"
"fmt"
"sort"
"strings"
"time"

"github.com/go-kit/kit/log"
@@ -75,7 +76,7 @@ func (d *Daemon) getResources(ctx context.Context) (map[string]resource.Resource
var globalReadOnly v6.ReadOnlyReason
err := d.WithClone(ctx, func(checkout *git.Checkout) error {
var err error
resources, err = d.Manifests.LoadManifests(checkout.Dir(), checkout.ManifestDir())
resources, err = d.Manifests.LoadManifests(checkout.Dir(), checkout.ManifestDirs())
return err
})

@@ -349,7 +350,7 @@ func (d *Daemon) updatePolicy(spec update.Spec, updates policy.Updates) updateFu
anythingAutomated = true
}
// find the service manifest
err := cluster.UpdateManifest(d.Manifests, working.ManifestDir(), serviceID, func(def []byte) ([]byte, error) {
err := cluster.UpdateManifest(d.Manifests, working.Dir(), working.ManifestDirs(), serviceID, func(def []byte) ([]byte, error) {
newDef, err := d.Manifests.UpdatePolicies(def, serviceID, u)
if err != nil {
result.Result[serviceID] = update.ControllerResult{
@@ -496,7 +497,7 @@ func (d *Daemon) JobStatus(ctx context.Context, jobID job.ID) (job.Status, error
if err != nil {
return errors.Wrap(err, "enumerating commit notes")
}
commits, err := d.Repo.CommitsBefore(ctx, "HEAD", d.GitConfig.Path)
commits, err := d.Repo.CommitsBefore(ctx, "HEAD", d.GitConfig.Paths...)
if err != nil {
return errors.Wrap(err, "checking revisions for status")
}
@@ -529,7 +530,7 @@ func (d *Daemon) JobStatus(ctx context.Context, jobID job.ID) (job.Status, error
// you'll get all the commits yet to be applied. If you send a hash
// and it's applied at or _past_ it, you'll get an empty list.
func (d *Daemon) SyncStatus(ctx context.Context, commitRef string) ([]string, error) {
commits, err := d.Repo.CommitsBetween(ctx, d.GitConfig.SyncTag, commitRef, d.GitConfig.Path)
commits, err := d.Repo.CommitsBetween(ctx, d.GitConfig.SyncTag, commitRef, d.GitConfig.Paths...)
if err != nil {
return nil, err
}
@@ -550,11 +551,15 @@ func (d *Daemon) GitRepoConfig(ctx context.Context, regenerate bool) (v6.GitConf

origin := d.Repo.Origin()
status, _ := d.Repo.Status()
path := ""
if len(d.GitConfig.Paths) > 0 {
path = strings.Join(d.GitConfig.Paths, ",")
}
return v6.GitConfig{
Remote: v6.GitRemoteConfig{
URL: origin.URL,
Branch: d.GitConfig.Branch,
Path: d.GitConfig.Path,
Path: path,
},
PublicSSHKey: publicSSHKey,
Status: status,
@@ -386,7 +386,8 @@ func TestDaemon_Release(t *testing.T) {
}
defer co.Clean()
// open a file
if file, err := os.Open(filepath.Join(co.ManifestDir(), "helloworld-deploy.yaml")); err == nil {
dirs := co.ManifestDirs()
if file, err := os.Open(filepath.Join(dirs[0], "helloworld-deploy.yaml")); err == nil {

// make sure it gets closed
defer file.Close()
@@ -430,7 +431,8 @@ func TestDaemon_PolicyUpdate(t *testing.T) {
return false
}
defer co.Clean()
m, err := d.Manifests.LoadManifests(co.Dir(), co.ManifestDir())
dirs := co.ManifestDirs()
m, err := d.Manifests.LoadManifests(co.Dir(), dirs)
if err != nil {
t.Fatalf("Error: %s", err.Error())
}
@@ -774,7 +776,8 @@ func (w *wait) ForImageTag(t *testing.T, d *Daemon, service, container, tag stri
}
defer co.Clean()

m, err := d.Manifests.LoadManifests(co.Dir(), co.ManifestDir())
dirs := co.ManifestDirs()
m, err := d.Manifests.LoadManifests(co.Dir(), dirs)
assert.NoError(t, err)

resources, err := d.Manifests.ParseManifests(m[service].Bytes())
@@ -96,6 +96,7 @@ func (r resources) IDs() (ids []flux.ResourceID) {
}

// getUnlockedAutomatedServices returns all the resources that are
// both automated, and not locked.
func (d *Daemon) getUnlockedAutomatedResources(ctx context.Context) (resources, error) {
resources, _, err := d.getResources(ctx)
if err != nil {
@@ -186,7 +186,7 @@ func (d *Daemon) doSync(logger log.Logger) (retErr error) {
}

// Get a map of all resources defined in the repo
allResources, err := d.Manifests.LoadManifests(working.Dir(), working.ManifestDir())
allResources, err := d.Manifests.LoadManifests(working.Dir(), working.ManifestDirs())
if err != nil {
return errors.Wrap(err, "loading resources from repo")
}
@@ -217,10 +217,10 @@ func (d *Daemon) doSync(logger log.Logger) (retErr error) {
var err error
ctx, cancel := context.WithTimeout(ctx, gitOpTimeout)
if oldTagRev != "" {
commits, err = d.Repo.CommitsBetween(ctx, oldTagRev, newTagRev, d.GitConfig.Path)
commits, err = d.Repo.CommitsBetween(ctx, oldTagRev, newTagRev, d.GitConfig.Paths...)
} else {
initialSync = true
commits, err = d.Repo.CommitsBefore(ctx, newTagRev, d.GitConfig.Path)
commits, err = d.Repo.CommitsBefore(ctx, newTagRev, d.GitConfig.Paths...)
}
cancel()
if err != nil {
@@ -240,7 +240,7 @@ func (d *Daemon) doSync(logger log.Logger) (retErr error) {
if err == nil && len(changedFiles) > 0 {
// We had some changed files, we're syncing a diff
// FIXME(michael): this won't be accurate when a file can have more than one resource
changedResources, err = d.Manifests.LoadManifests(working.Dir(), changedFiles[0], changedFiles[1:]...)
changedResources, err = d.Manifests.LoadManifests(working.Dir(), changedFiles)
}
cancel()
if err != nil {
@@ -135,7 +135,7 @@ func TestPullAndSync_InitialSync(t *testing.T) {
// It creates the tag at HEAD
if err := d.Repo.Refresh(context.Background()); err != nil {
t.Errorf("pulling sync tag: %v", err)
} else if revs, err := d.Repo.CommitsBefore(context.Background(), gitSyncTag, gitPath); err != nil {
} else if revs, err := d.Repo.CommitsBefore(context.Background(), gitSyncTag); err != nil {
t.Errorf("finding revisions before sync tag: %v", err)
} else if len(revs) <= 0 {
t.Errorf("Found no revisions before the sync tag")
@@ -197,12 +197,12 @@ func TestDoSync_NoNewCommits(t *testing.T) {
}

// It doesn't move the tag
oldRevs, err := d.Repo.CommitsBefore(ctx, gitSyncTag, gitPath)
oldRevs, err := d.Repo.CommitsBefore(ctx, gitSyncTag)
if err != nil {
t.Fatal(err)
}

if revs, err := d.Repo.CommitsBefore(ctx, gitSyncTag, gitPath); err != nil {
if revs, err := d.Repo.CommitsBefore(ctx, gitSyncTag); err != nil {
t.Errorf("finding revisions before sync tag: %v", err)
} else if !reflect.DeepEqual(revs, oldRevs) {
t.Errorf("Should have kept the sync tag at HEAD")
@@ -230,7 +230,8 @@ func TestDoSync_WithNewCommit(t *testing.T) {
return err
}
// Push some new changes
err = cluster.UpdateManifest(k8s, checkout.ManifestDir(), flux.MustParseResourceID("default:deployment/helloworld"), func(def []byte) ([]byte, error) {
dirs := checkout.ManifestDirs()
err = cluster.UpdateManifest(k8s, checkout.Dir(), dirs, flux.MustParseResourceID("default:deployment/helloworld"), func(def []byte) ([]byte, error) {
// A simple modification so we have changes to push
return []byte(strings.Replace(string(def), "replicas: 5", "replicas: 4", -1)), nil
})
@@ -300,7 +301,7 @@ func TestDoSync_WithNewCommit(t *testing.T) {
defer cancel()
if err := d.Repo.Refresh(ctx); err != nil {
t.Errorf("pulling sync tag: %v", err)
} else if revs, err := d.Repo.CommitsBetween(ctx, oldRevision, gitSyncTag, gitPath); err != nil {
} else if revs, err := d.Repo.CommitsBetween(ctx, oldRevision, gitSyncTag); err != nil {
t.Errorf("finding revisions before sync tag: %v", err)
} else if len(revs) <= 0 {
t.Errorf("Should have moved sync tag forward")
@@ -25,7 +25,8 @@ func TestCommit(t *testing.T) {
defer cleanup()

for file, _ := range testfiles.Files {
path := filepath.Join(checkout.ManifestDir(), file)
dirs := checkout.ManifestDirs()
path := filepath.Join(dirs[0], file)
if err := ioutil.WriteFile(path, []byte("FIRST CHANGE"), 0666); err != nil {
t.Fatal(err)
}
@@ -45,7 +46,7 @@ func TestCommit(t *testing.T) {
t.Error(err)
}

commits, err := repo.CommitsBefore(ctx, "HEAD", "")
commits, err := repo.CommitsBefore(ctx, "HEAD")

if err != nil {
t.Fatal(err)
@@ -107,8 +108,9 @@ func TestCheckout(t *testing.T) {
}

changedFile := ""
dirs := checkout.ManifestDirs()
for file, _ := range testfiles.Files {
path := filepath.Join(checkout.ManifestDir(), file)
path := filepath.Join(dirs[0], file)
if err := ioutil.WriteFile(path, []byte("FIRST CHANGE"), 0666); err != nil {
t.Fatal(err)
}
@@ -120,7 +122,7 @@ func TestCheckout(t *testing.T) {
t.Fatal(err)
}

path := filepath.Join(checkout.ManifestDir(), changedFile)
path := filepath.Join(dirs[0], changedFile)
if err := ioutil.WriteFile(path, []byte("SECOND CHANGE"), 0666); err != nil {
t.Fatal(err)
}
@@ -135,7 +137,7 @@ func TestCheckout(t *testing.T) {
}

check := func(c *git.Checkout) {
contents, err := ioutil.ReadFile(filepath.Join(c.ManifestDir(), changedFile))
contents, err := ioutil.ReadFile(filepath.Join(dirs[0], changedFile))
if err != nil {
t.Fatal(err)
}

0 comments on commit 670fce0

Please sign in to comment.
You can’t perform that action at this time.