From 2828c38c0c0f0862cef6cd6e5b6d36e17e4be8db Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Fri, 14 Mar 2025 17:17:06 +0000 Subject: [PATCH 01/12] refector WorkTreeLink and use appropriate field names --- pkg/mirror/helper.go | 2 +- pkg/mirror/repository.go | 44 +++++++++++++++++------------------ pkg/mirror/repository_test.go | 26 ++++++++++----------- pkg/mirror/worktree.go | 8 +++---- pkg/mirror/z_e2e_test.go | 22 +++++++++--------- 5 files changed, 49 insertions(+), 53 deletions(-) diff --git a/pkg/mirror/helper.go b/pkg/mirror/helper.go index c48a6be..7a04ae8 100644 --- a/pkg/mirror/helper.go +++ b/pkg/mirror/helper.go @@ -125,7 +125,7 @@ func removeDirContents(dir string, log *slog.Logger) error { }) } -// removeDirContents iterated the specified dir and removes entries +// removeDirContentsIf iterated the specified dir and removes entries // if given function returns true for the given entry func removeDirContentsIf(dir string, log *slog.Logger, fn func(fi os.FileInfo) (bool, error)) error { dirents, err := os.ReadDir(dir) diff --git a/pkg/mirror/repository.go b/pkg/mirror/repository.go index 85621b3..9260e9b 100644 --- a/pkg/mirror/repository.go +++ b/pkg/mirror/repository.go @@ -52,7 +52,7 @@ type Repository struct { lock lock.RWMutex // repository will be locked during mirror gitURL *giturl.URL // parsed remote git URL remote string // remote repo to mirror - root string // absolute path to the root where repo directory createdabsolute path to the root where repo directory created + root string // absolute path to the root where repo directory created dir string // absolute path to the repo directory interval time.Duration // how long to wait between mirrors mirrorTimeout time.Duration // the total time allowed for the mirror loop @@ -123,40 +123,38 @@ func NewRepository(repoConf RepositoryConfig, envs []string, log *slog.Logger) ( } for _, wtc := range repoConf.Worktrees { - if err := repo.AddWorktreeLink(wtc.Link, wtc.Ref, wtc.Pathspec); err != nil { + if err := repo.AddWorktreeLink(wtc); err != nil { return nil, fmt.Errorf("unable to create worktree link err:%w", err) } } return repo, nil } -// AddWorktreeLink adds add workTree link to the mirror repository. -func (r *Repository) AddWorktreeLink(link, ref, pathspec string) error { - if link == "" { +// AddWorktreeLink adds workTree link to the mirror repository. +func (r *Repository) AddWorktreeLink(wtc WorktreeConfig) error { + if wtc.Link == "" { return fmt.Errorf("symlink path cannot be empty") } - if v, ok := r.workTreeLinks[link]; ok { - return fmt.Errorf("worktree with given link already exits link:%s ref:%s", v.link, v.ref) + if v, ok := r.workTreeLinks[wtc.Link]; ok { + return fmt.Errorf("worktree with given link already exits link:%s ref:%s", v.linkAbs, v.ref) } - linkAbs := absLink(r.root, link) + linkAbs := absLink(r.root, wtc.Link) - if ref == "" { - ref = "HEAD" + if wtc.Ref == "" { + wtc.Ref = "HEAD" } - _, linkFile := splitAbs(link) - wt := &WorkTreeLink{ - name: linkFile, - link: linkAbs, - ref: ref, - pathspec: pathspec, - log: r.log.With("worktree", linkFile), + link: wtc.Link, + linkAbs: linkAbs, + ref: wtc.Ref, + pathspec: wtc.Pathspec, + log: r.log.With("worktree", wtc.Link), } - r.workTreeLinks[link] = wt + r.workTreeLinks[wtc.Link] = wt return nil } @@ -447,7 +445,7 @@ func (r *Repository) Mirror(ctx context.Context) error { // so always ensure worktree even if nothing fetched for _, wl := range r.workTreeLinks { if err := r.ensureWorktreeLink(ctx, wl); err != nil { - return fmt.Errorf("unable to ensure worktree links repo:%s link:%s err:%w", r.gitURL.Repo, wl.name, err) + return fmt.Errorf("unable to ensure worktree links repo:%s link:%s err:%w", r.gitURL.Repo, wl.link, err) } } @@ -664,7 +662,7 @@ func (r *Repository) ensureWorktreeLink(ctx context.Context, wl *WorkTreeLink) e // get remote hash from mirrored repo for the worktree link remoteHash, err := r.hash(ctx, wl.ref, wl.pathspec) if err != nil { - return fmt.Errorf("unable to get hash for worktree:%s err:%w", wl.name, err) + return fmt.Errorf("unable to get hash for worktree:%s err:%w", wl.link, err) } var currentHash, currentPath string @@ -714,10 +712,10 @@ func (r *Repository) ensureWorktreeLink(ctx context.Context, wl *WorkTreeLink) e wl.log.Info("worktree update required", "remoteHash", remoteHash, "currentHash", currentHash) newPath, err := r.createWorktree(ctx, wl, remoteHash) if err != nil { - return fmt.Errorf("unable to create worktree for '%s' err:%w", wl.name, err) + return fmt.Errorf("unable to create worktree for '%s' err:%w", wl.link, err) } - if err = publishSymlink(wl.link, newPath); err != nil { + if err = publishSymlink(wl.linkAbs, newPath); err != nil { return fmt.Errorf("unable to publish symlink err:%w", err) } @@ -834,7 +832,7 @@ func (r *Repository) removeStaleWorktrees() (int, error) { for _, wt := range r.workTreeLinks { t, err := wt.currentWorktree() if err != nil { - r.log.Error("unable to read worktree link", "worktree", wt.name, "err", err) + r.log.Error("unable to read worktree link", "worktree", wt.link, "err", err) continue } if t != "" { diff --git a/pkg/mirror/repository_test.go b/pkg/mirror/repository_test.go index 29a0244..e26762f 100644 --- a/pkg/mirror/repository_test.go +++ b/pkg/mirror/repository_test.go @@ -132,35 +132,33 @@ func TestRepo_AddWorktreeLink(t *testing.T) { } type args struct { - link string - ref string - pathspec string + wtc WorktreeConfig } tests := []struct { name string args args wantErr bool }{ - {"all-valid", args{"link", "master", ""}, false}, - {"all-valid-with-path", args{"link2", "other-branch", "path"}, false}, - {"duplicate-link", args{"link", "master", ""}, true}, - {"no-link", args{"", "master", ""}, true}, - {"no-ref", args{"link3", "", ""}, false}, - {"absLink", args{"/tmp/link", "tag", ""}, false}, + {"all-valid", args{wtc: WorktreeConfig{"link", "master", ""}}, false}, + {"all-valid-with-path", args{wtc: WorktreeConfig{"link2", "other-branch", "path"}}, false}, + {"duplicate-link", args{wtc: WorktreeConfig{"link", "master", ""}}, true}, + {"no-link", args{wtc: WorktreeConfig{"", "master", ""}}, true}, + {"no-ref", args{wtc: WorktreeConfig{"link3", "", ""}}, false}, + {"absLink", args{wtc: WorktreeConfig{"/tmp/link", "tag", ""}}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := r.AddWorktreeLink(tt.args.link, tt.args.ref, tt.args.pathspec); (err != nil) != tt.wantErr { + if err := r.AddWorktreeLink(tt.args.wtc); (err != nil) != tt.wantErr { t.Errorf("Repo.AddWorktreeLink() error = %v, wantErr %v", err, tt.wantErr) } }) } // compare all worktree links want := map[string]*WorkTreeLink{ - "link": {name: "link", link: "/tmp/root/link", ref: "master"}, - "link2": {name: "link2", link: "/tmp/root/link2", ref: "other-branch", pathspec: "path"}, - "link3": {name: "link3", link: "/tmp/root/link3", ref: "HEAD"}, - "/tmp/link": {name: "link", link: "/tmp/link", ref: "tag"}, + "link": {link: "link", linkAbs: "/tmp/root/link", ref: "master"}, + "link2": {link: "link2", linkAbs: "/tmp/root/link2", ref: "other-branch", pathspec: "path"}, + "link3": {link: "link3", linkAbs: "/tmp/root/link3", ref: "HEAD"}, + "/tmp/link": {link: "link", linkAbs: "/tmp/link", ref: "tag"}, } if diff := cmp.Diff(want, r.workTreeLinks, cmpopts.IgnoreFields(WorkTreeLink{}, "log"), cmp.AllowUnexported(WorkTreeLink{})); diff != "" { t.Errorf("Repo.AddWorktreeLink() worktreelinks mismatch (-want +got):\n%s", diff) diff --git a/pkg/mirror/worktree.go b/pkg/mirror/worktree.go index 4ea7f51..904d95a 100644 --- a/pkg/mirror/worktree.go +++ b/pkg/mirror/worktree.go @@ -9,8 +9,8 @@ import ( ) type WorkTreeLink struct { - name string // link file name might not be unique only use it for logging - link string // the path at which to create a symlink to the worktree dir + link string // link name as its specified in config, might not be unique only use it for logging + linkAbs string // the path at which to create a symlink to the worktree dir ref string // the ref of the worktree pathspec string // pathspec of the dirs to checkout log *slog.Logger @@ -20,13 +20,13 @@ type WorkTreeLink struct { // two worktree links can be on same ref but with diff pathspecs // hence we cant just use tree hash as path func (w *WorkTreeLink) worktreeDirName(hash string) string { - parts := strings.Split(strings.Trim(w.link, "/"), "/") + parts := strings.Split(strings.Trim(w.linkAbs, "/"), "/") return parts[len(parts)-1] + "-" + hash[:7] } // currentWorktree reads symlink path of the given worktree link func (wl *WorkTreeLink) currentWorktree() (string, error) { - return readAbsLink(wl.link) + return readAbsLink(wl.linkAbs) } // workTreeHash returns the hash of the given revision and for the path if specified. diff --git a/pkg/mirror/z_e2e_test.go b/pkg/mirror/z_e2e_test.go index 5cffebb..7f11166 100644 --- a/pkg/mirror/z_e2e_test.go +++ b/pkg/mirror/z_e2e_test.go @@ -184,7 +184,7 @@ func Test_mirror_head_and_main(t *testing.T) { repo := mustCreateRepoAndMirror(t, upstream, root, link1, ref1) // add worktree for HEAD - if err := repo.AddWorktreeLink(link2, ref2, ""); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link2, ref2, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } // mirror again for 2nd worktree @@ -271,7 +271,7 @@ func Test_mirror_other_branch(t *testing.T) { repo := mustCreateRepoAndMirror(t, upstream, root, link1, ref1) // add 2nd worktree - if err := repo.AddWorktreeLink(link2, ref2, ""); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link2, ref2, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } // mirror again for 2nd worktree @@ -333,10 +333,10 @@ func Test_mirror_with_pathspec(t *testing.T) { repo := mustCreateRepoAndMirror(t, upstream, root, link1, ref1) // add worktree for HEAD - if err := repo.AddWorktreeLink(link2, ref2, pathSpec2); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link2, ref2, pathSpec2}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } - if err := repo.AddWorktreeLink(link3, ref3, pathSpec3); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link3, ref3, pathSpec3}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } // mirror again for 2nd worktree @@ -426,7 +426,7 @@ func Test_mirror_switch_branch_after_restart(t *testing.T) { repo1 := mustCreateRepoAndMirror(t, upstream, root, link1, ref1) // add 2nd worktree - if err := repo1.AddWorktreeLink(link2, ref2, ""); err != nil { + if err := repo1.AddWorktreeLink(WorktreeConfig{link2, ref2, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } // mirror again for 2nd worktree @@ -442,7 +442,7 @@ func Test_mirror_switch_branch_after_restart(t *testing.T) { repo2 := mustCreateRepoAndMirror(t, upstream, root, link1, ref2) // add 2nd worktree - if err := repo2.AddWorktreeLink(link2, ref1, ""); err != nil { + if err := repo2.AddWorktreeLink(WorktreeConfig{link2, ref1, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } // mirror again for 2nd worktree @@ -502,7 +502,7 @@ func Test_mirror_tag_sha(t *testing.T) { repo := mustCreateRepoAndMirror(t, upstream, root, link1, ref1) // add 2nd worktree - if err := repo.AddWorktreeLink(link2, ref2, ""); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link2, ref2, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } // mirror again for 2nd worktree @@ -1229,7 +1229,7 @@ func Test_mirror_loop(t *testing.T) { repo := mustCreateRepoAndMirror(t, upstream, root, link1, ref1) // add worktree for HEAD - if err := repo.AddWorktreeLink(link2, ref2, ""); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link2, ref2, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } @@ -1322,7 +1322,7 @@ func Test_RepoPool_Success(t *testing.T) { // add worktree // we will verify this worktree in next mirror loop - if err := rp.AddWorktreeLink(remote1, "link3", "", ""); err != nil { + if err := rp.AddWorktreeLink(remote1, WorktreeConfig{"link3", "", ""}); err != nil { t.Fatalf("unexpected err:%s", err) } @@ -1524,7 +1524,7 @@ func Test_RepoPool_Error(t *testing.T) { t.Errorf("unexpected err:%s", err) } - if err := rp.AddRepository(repo1); err == nil { + if err := rp.AddRepository(RepositoryConfig{Remote: repo1.remote}); err == nil { t.Errorf("unexpected success for non existing repo") } else if err != ErrExist { t.Errorf("error mismatch got:%s want:%s", err, ErrNotExist) @@ -1574,7 +1574,7 @@ func mustCreateRepoAndMirror(t *testing.T, upstream, root, link, ref string) *Re t.Fatalf("unable to create new repo error: %v", err) } if link != "" { - if err := repo.AddWorktreeLink(link, ref, ""); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link, ref, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } } From 26f57f2d096dd78b50850ac27cb0cb37e6c0f055 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 14:35:46 +0000 Subject: [PATCH 02/12] stop mirror loop using context cancel --- pkg/mirror/example_noworktree_test.go | 2 +- pkg/mirror/example_worktree_test.go | 2 +- pkg/mirror/repo_pool.go | 8 +++++--- pkg/mirror/z_e2e_test.go | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/mirror/example_noworktree_test.go b/pkg/mirror/example_noworktree_test.go index 5957d90..26dc6df 100644 --- a/pkg/mirror/example_noworktree_test.go +++ b/pkg/mirror/example_noworktree_test.go @@ -53,7 +53,7 @@ repositories: } // start mirror Loop - repos.StartLoop() + repos.StartLoop(ctx) hash, err := repos.Hash(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "") if err != nil { diff --git a/pkg/mirror/example_worktree_test.go b/pkg/mirror/example_worktree_test.go index 2b1c3c5..99a82a5 100644 --- a/pkg/mirror/example_worktree_test.go +++ b/pkg/mirror/example_worktree_test.go @@ -56,7 +56,7 @@ repositories: } // start mirror Loop - repos.StartLoop() + repos.StartLoop(ctx) hash, err := repos.Hash(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "") if err != nil { diff --git a/pkg/mirror/repo_pool.go b/pkg/mirror/repo_pool.go index 20914aa..a250cd9 100644 --- a/pkg/mirror/repo_pool.go +++ b/pkg/mirror/repo_pool.go @@ -97,13 +97,15 @@ func (rp *RepoPool) Mirror(ctx context.Context, remote string) error { // StartLoop will start mirror loop on all repositories // if its not already started -func (rp *RepoPool) StartLoop() { +func (rp *RepoPool) StartLoop(ctx context.Context) { + rp.lock.RLock() + defer rp.lock.RUnlock() + for _, repo := range rp.repos { if !repo.running { - go repo.StartLoop(context.TODO()) + go repo.StartLoop(ctx) continue } - rp.log.Info("start loop is already running", "repo", repo.gitURL.Repo) } } diff --git a/pkg/mirror/z_e2e_test.go b/pkg/mirror/z_e2e_test.go index 7f11166..d66bcdc 100644 --- a/pkg/mirror/z_e2e_test.go +++ b/pkg/mirror/z_e2e_test.go @@ -1368,11 +1368,11 @@ func Test_RepoPool_Success(t *testing.T) { t.Log("TEST-2: forward both upstream and test mirrors") // start mirror loop - rp.StartLoop() + rp.StartLoop(t.Context()) time.Sleep(time.Second) // start mirror loop again this should be no op - rp.StartLoop() + rp.StartLoop(t.Context()) fileU1SHA2 := mustCommit(t, upstream1, "file", t.Name()+"-u1-main-2") fileU2SHA2 := mustCommit(t, upstream2, "file", t.Name()+"-u2-main-2") @@ -1497,7 +1497,7 @@ func Test_RepoPool_Error(t *testing.T) { t.Fatalf("unexpected error: %v", err) } // start mirror loop - rp.StartLoop() + rp.StartLoop(t.Context()) time.Sleep(time.Second) From 88f03fa4fd5d0ce8df162e6272d194e5ef962d8c Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 14:39:44 +0000 Subject: [PATCH 03/12] use standard function name --- pkg/giturl/git_url.go | 6 +++--- pkg/mirror/repo_pool.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/giturl/git_url.go b/pkg/giturl/git_url.go index 3af0668..62aeb5d 100644 --- a/pkg/giturl/git_url.go +++ b/pkg/giturl/git_url.go @@ -98,10 +98,10 @@ func Parse(rawURL string) (*URL, error) { return gURL, nil } -// SameURL returns whether or not the two parsed git URLs are equivalent. +// Equals returns whether or not the two parsed git URLs are equivalent. // git URLs can be represented in multiple schemes so if host, path and repo name // of URLs are same then those URLs are for the same remote repository -func SameURL(lURL, rURL *URL) bool { +func (lURL *URL) Equals(rURL *URL) bool { return lURL.Host == rURL.Host && lURL.Path == rURL.Path && (lURL.Repo == rURL.Repo || @@ -119,7 +119,7 @@ func SameRawURL(lRepo, rRepo string) (bool, error) { return false, err } - return SameURL(lURL, rURL), nil + return lURL.Equals(rURL), nil } // IsSCPURL returns true if supplied URL is scp-like syntax diff --git a/pkg/mirror/repo_pool.go b/pkg/mirror/repo_pool.go index a250cd9..1009025 100644 --- a/pkg/mirror/repo_pool.go +++ b/pkg/mirror/repo_pool.go @@ -117,7 +117,7 @@ func (rp *RepoPool) Repository(remote string) (*Repository, error) { } for _, repo := range rp.repos { - if giturl.SameURL(repo.gitURL, gitURL) { + if repo.gitURL.Equals(gitURL) { return repo, nil } } From 481de78ee322bba5a63babce7093a0c5166d220b Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 14:44:52 +0000 Subject: [PATCH 04/12] add graceful shutdown and worktree cleanup --- pkg/mirror/repository.go | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/pkg/mirror/repository.go b/pkg/mirror/repository.go index 9260e9b..db036d4 100644 --- a/pkg/mirror/repository.go +++ b/pkg/mirror/repository.go @@ -5,6 +5,7 @@ import ( "fmt" "io/fs" "log/slog" + "maps" "os" "os/exec" "path/filepath" @@ -158,6 +159,13 @@ func (r *Repository) AddWorktreeLink(wtc WorktreeConfig) error { return nil } +func (r *Repository) WorktreeLinks() map[string]*WorkTreeLink { + r.lock.RLock() + defer r.lock.RUnlock() + + return maps.Clone(r.workTreeLinks) +} + // Hash returns the hash of the given revision and for the path if specified. func (r *Repository) Hash(ctx context.Context, ref, path string) (string, error) { r.lock.RLock() @@ -417,6 +425,13 @@ func (r *Repository) StartLoop(ctx context.Context) { } } +// StopLoop stops sync loop gracefully +func (r *Repository) StopLoop() { + r.stop <- true + <-r.stopped + r.log.Info("repository mirror loop stopped") +} + // Mirror will run mirror loop of the repository // 1. init and validate if existing repo dir // 2. fetch remote @@ -462,6 +477,38 @@ func (r *Repository) Mirror(ctx context.Context) error { return nil } +// RemoveWorktreeLink removes workTree link from the mirror repository. +// it will remove published link as well even if it failed to remove worktree +func (r *Repository) RemoveWorktreeLink(link string) error { + r.lock.Lock() + defer r.lock.Unlock() + + wl, ok := r.workTreeLinks[link] + if !ok { + return fmt.Errorf("worktree with given link not found") + } + + defer func() { + // remove worktree link from repo object + delete(r.workTreeLinks, link) + // remove published link + if err := os.Remove(wl.linkAbs); err != nil { + r.log.Error("unable to remove published link", "err", err) + } + }() + + wtPath, err := wl.currentWorktree() + if err != nil { + return err + } + + if err := r.removeWorktree(context.TODO(), wtPath); err != nil { + return err + } + + return nil +} + // worktreesRoot returns abs path for all the worktrees of the repo // git uses `worktrees` folder for its on use hence we are using `.worktrees` func (r *Repository) worktreesRoot() string { From 001189b828bf3a5ea13711928b8d6fc27913f2d7 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 14:57:28 +0000 Subject: [PATCH 05/12] support adding respository and worktrees concurrently --- pkg/mirror/config.go | 14 ++++++++++ pkg/mirror/repo_pool.go | 60 ++++++++++++++++++++++++----------------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/pkg/mirror/config.go b/pkg/mirror/config.go index e749416..d34194f 100644 --- a/pkg/mirror/config.go +++ b/pkg/mirror/config.go @@ -183,6 +183,20 @@ func (rpc *RepoPoolConfig) ValidateLinkPaths() error { } +// ValidateAndApplyDefaults will validate link paths and default and apply defaults +func (conf *RepoPoolConfig) ValidateAndApplyDefaults() error { + if err := conf.ValidateDefaults(); err != nil { + return err + } + + if err := conf.ValidateLinkPaths(); err != nil { + return err + } + + conf.ApplyDefaults() + return nil +} + // gitSSHCommand returns the environment variable to be used for configuring // git over ssh. func (a Auth) gitSSHCommand() string { diff --git a/pkg/mirror/repo_pool.go b/pkg/mirror/repo_pool.go index 1009025..bc14ead 100644 --- a/pkg/mirror/repo_pool.go +++ b/pkg/mirror/repo_pool.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "log/slog" + "os" + "slices" "time" "github.com/utilitywarehouse/git-mirror/pkg/giturl" + "github.com/utilitywarehouse/git-mirror/pkg/lock" ) var ( @@ -18,37 +21,27 @@ var ( // it provides simple wrapper around Repository methods. // A RepoPool is safe for concurrent use by multiple goroutines. type RepoPool struct { - log *slog.Logger - repos []*Repository + lock lock.RWMutex + log *slog.Logger + repos []*Repository + commonENVs []string } // NewRepoPool will create mirror repositories based on given config. // Remote repo will not be mirrored until either Mirror() or StartLoop() is called func NewRepoPool(conf RepoPoolConfig, log *slog.Logger, commonENVs []string) (*RepoPool, error) { - if err := conf.ValidateDefaults(); err != nil { + if err := conf.ValidateAndApplyDefaults(); err != nil { return nil, err } - if err := conf.ValidateLinkPaths(); err != nil { - return nil, err - } - - conf.ApplyDefaults() - if log == nil { log = slog.Default() } - rp := &RepoPool{log: log} + rp := &RepoPool{log: log, commonENVs: commonENVs} for _, repoConf := range conf.Repositories { - - repo, err := NewRepository(repoConf, commonENVs, log) - if err != nil { - return nil, err - } - - if err := rp.AddRepository(repo); err != nil { + if err := rp.AddRepository(repoConf); err != nil { return nil, err } } @@ -58,11 +51,19 @@ func NewRepoPool(conf RepoPoolConfig, log *slog.Logger, commonENVs []string) (*R // AddRepository will add given repository to repoPool. // Remote repo will not be mirrored until either Mirror() or StartLoop() is called -func (rp *RepoPool) AddRepository(repo *Repository) error { - if repo, _ := rp.Repository(repo.remote); repo != nil { +func (rp *RepoPool) AddRepository(repoConf RepositoryConfig) error { + remoteURL := giturl.NormaliseURL(repoConf.Remote) + if repo, _ := rp.Repository(remoteURL); repo != nil { return ErrExist } + rp.lock.Lock() + defer rp.lock.Unlock() + + repo, err := NewRepository(repoConf, rp.commonENVs, rp.log) + if err != nil { + return err + } rp.repos = append(rp.repos, repo) return nil @@ -73,6 +74,9 @@ func (rp *RepoPool) AddRepository(repo *Repository) error { // Ideally MirrorAll should be used for the first mirror cycle to ensure repositories are // successfully mirrored func (rp *RepoPool) MirrorAll(ctx context.Context, timeout time.Duration) error { + rp.lock.RLock() + defer rp.lock.RUnlock() + for _, repo := range rp.repos { mCtx, cancel := context.WithTimeout(ctx, timeout) err := repo.Mirror(mCtx) @@ -116,6 +120,9 @@ func (rp *RepoPool) Repository(remote string) (*Repository, error) { return nil, err } + rp.lock.RLock() + defer rp.lock.RUnlock() + for _, repo := range rp.repos { if repo.gitURL.Equals(gitURL) { return repo, nil @@ -125,25 +132,28 @@ func (rp *RepoPool) Repository(remote string) (*Repository, error) { } // AddWorktreeLink is wrapper around repositories AddWorktreeLink method -func (rp *RepoPool) AddWorktreeLink(remote string, link, ref, pathspec string) error { +func (rp *RepoPool) AddWorktreeLink(remote string, wt WorktreeConfig) error { + rp.lock.RLock() + defer rp.lock.RUnlock() + repo, err := rp.Repository(remote) if err != nil { return err } - if err := rp.validateLinkPath(repo, link); err != nil { + if err := rp.validateLinkPath(repo, wt.Link); err != nil { return err } - return repo.AddWorktreeLink(link, ref, pathspec) + return repo.AddWorktreeLink(wt) } func (rp *RepoPool) validateLinkPath(repo *Repository, link string) error { newAbsLink := absLink(repo.root, link) for _, r := range rp.repos { - for _, wl := range r.workTreeLinks { - if wl.link == newAbsLink { + for _, wl := range r.WorktreeLinks() { + if wl.linkAbs == newAbsLink { return fmt.Errorf("repo with overlapping abs link path found repo:%s path:%s", - r.gitURL.Repo, wl.link) + r.gitURL.Repo, wl.linkAbs) } } } From 410ba8faa56c43161519e42af273483f0d41ffc8 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 14:59:34 +0000 Subject: [PATCH 06/12] support removing respository and worktrees concurrently --- pkg/mirror/repo_pool.go | 47 +++++++++++++++++++++++++++++++++++++ pkg/mirror/z_e2e_test.go | 50 +++++++++++++++++++++++++++++++++++----- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/pkg/mirror/repo_pool.go b/pkg/mirror/repo_pool.go index bc14ead..ae5890b 100644 --- a/pkg/mirror/repo_pool.go +++ b/pkg/mirror/repo_pool.go @@ -131,6 +131,17 @@ func (rp *RepoPool) Repository(remote string) (*Repository, error) { return nil, ErrNotExist } +func (rp *RepoPool) RepositoriesRemote() []string { + rp.lock.RLock() + defer rp.lock.RUnlock() + + var urls []string + for _, repo := range rp.repos { + urls = append(urls, repo.remote) + } + return urls +} + // AddWorktreeLink is wrapper around repositories AddWorktreeLink method func (rp *RepoPool) AddWorktreeLink(remote string, wt WorktreeConfig) error { rp.lock.RLock() @@ -161,6 +172,42 @@ func (rp *RepoPool) validateLinkPath(repo *Repository, link string) error { return nil } +// RemoveWorktreeLink is wrapper around repositories RemoveWorktreeLink method +func (rp *RepoPool) RemoveWorktreeLink(remote string, wtLink string) error { + repo, err := rp.Repository(remote) + if err != nil { + return err + } + return repo.RemoveWorktreeLink(wtLink) +} + +// RemoveRepository will remove given repository from the repoPool. +func (rp *RepoPool) RemoveRepository(remote string) error { + rp.lock.Lock() + defer rp.lock.Unlock() + + for i, repo := range rp.repos { + if repo.remote == remote { + rp.log.Info("removing repository mirror", "remote", repo.remote) + + rp.repos = slices.Delete(rp.repos, i, i+1) + + repo.StopLoop() + + // remove all published links + for _, wt := range repo.WorktreeLinks() { + if err := os.Remove(wt.linkAbs); err != nil { + rp.log.Error("unable to remove published link", "err", err) + } + } + + return os.RemoveAll(repo.dir) + } + } + + return ErrNotExist +} + // Hash is wrapper around repositories hash method func (rp *RepoPool) Hash(ctx context.Context, remote, ref, path string) (string, error) { repo, err := rp.Repository(remote) diff --git a/pkg/mirror/z_e2e_test.go b/pkg/mirror/z_e2e_test.go index d66bcdc..19e1752 100644 --- a/pkg/mirror/z_e2e_test.go +++ b/pkg/mirror/z_e2e_test.go @@ -215,6 +215,17 @@ func Test_mirror_head_and_main(t *testing.T) { } assertLinkedFile(t, root, link1, "file", t.Name()+"-1") assertLinkedFile(t, root, link2, "file", t.Name()+"-1") + + // remove worktrees + if err := repo.RemoveWorktreeLink(link2); err != nil { + t.Errorf("unable to remove worktree error: %v", err) + } + assertMissingLink(t, root, link2) + + if err := repo.RemoveWorktreeLink(link1); err != nil { + t.Errorf("unable to remove worktree error: %v", err) + } + assertMissingLink(t, root, link1) } func Test_mirror_bad_ref(t *testing.T) { @@ -310,6 +321,17 @@ func Test_mirror_other_branch(t *testing.T) { } assertLinkedFile(t, root, link1, "file", t.Name()+"-main-1") assertLinkedFile(t, root, link2, "file", t.Name()+"-other-1") + + // remove worktrees + if err := repo.RemoveWorktreeLink(link2); err != nil { + t.Errorf("unable to remove worktree error: %v", err) + } + assertMissingLink(t, root, link2) + + if err := repo.RemoveWorktreeLink(link1); err != nil { + t.Errorf("unable to remove worktree error: %v", err) + } + assertMissingLink(t, root, link1) } func Test_mirror_with_pathspec(t *testing.T) { @@ -1268,12 +1290,9 @@ func Test_mirror_loop(t *testing.T) { assertLinkedFile(t, root, link2, "file", t.Name()+"-1") // STOP mirror loop - repo.stop <- true - - time.Sleep(testInterval) - - if repo.running != false { - t.Errorf("repo still running after sending stop signal") + repo.StopLoop() + if repo.running { + t.Errorf("repo still running after calling StopLoop") } } @@ -1440,6 +1459,12 @@ func Test_RepoPool_Success(t *testing.T) { assertLinkedFile(t, root, "link3", "file", t.Name()+"-u1-main-1") assertLinkedFile(t, root, "link2", "file", t.Name()+"-u2-main-1") + // remove worktrees + if err := rp.RemoveWorktreeLink(remote2, "link2"); err != nil { + t.Errorf("unable to remove worktree error: %v", err) + } + assertMissingLink(t, root, "link2") + if cloneSHA, err := rp.Clone(txtCtx, remote1, tempClone, testMainBranch, "", false); err != nil { t.Fatalf("unexpected error %s", err) } else { @@ -1457,6 +1482,19 @@ func Test_RepoPool_Success(t *testing.T) { } assertFile(t, filepath.Join(tempClone, "file"), t.Name()+"-u2-main-1") } + + // remove repository + repo, _ := rp.Repository(remote1) + if err := rp.RemoveRepository(remote1); err != nil { + t.Errorf("unable to remove repository error: %v", err) + } + if len(rp.repos) > 1 { + t.Errorf("there should be only 1 repo in repoPool now") + } + // once repo is removed public link should be removed + assertMissingLink(t, root, "link1") + // root dir should be empty + assertMissingLink(t, repo.dir, "") } func Test_RepoPool_Error(t *testing.T) { From 44da98497ad07617717c77ddab3588471ff29cd5 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 16:05:10 +0000 Subject: [PATCH 07/12] add git-mirror config watcher --- config.go | 205 +++++++++++++++++++++++++++++++++++++++++ config_test.go | 186 +++++++++++++++++++++++++++++++++++++ main.go | 75 +++++---------- pkg/mirror/worktree.go | 6 ++ 4 files changed, 421 insertions(+), 51 deletions(-) create mode 100644 config.go create mode 100644 config_test.go diff --git a/config.go b/config.go new file mode 100644 index 0000000..c284523 --- /dev/null +++ b/config.go @@ -0,0 +1,205 @@ +package main + +import ( + "errors" + "os" + "path" + "time" + + "github.com/utilitywarehouse/git-mirror/pkg/giturl" + "github.com/utilitywarehouse/git-mirror/pkg/mirror" + "gopkg.in/yaml.v3" +) + +const ( + defaultGitGC = "always" + defaultInterval = 30 * time.Second + defaultMirrorTimeout = 2 * time.Minute + defaultSSHKeyPath = "/etc/git-secret/ssh" + defaultSSHKnownHostsPath = "/etc/git-secret/known_hosts" +) + +var defaultRoot = path.Join(os.TempDir(), "git-mirror", "src") + +// WatchConfig polls the config file every interval and reloads if modified +func WatchConfig(path string, interval time.Duration, onChange func(*mirror.RepoPoolConfig)) { + var lastModTime time.Time + + // Load initial config + config, err := parseConfigFile(path) + if err != nil { + logger.Error("failed to load config", "err", err) + } else { + onChange(config) + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for range ticker.C { + fileInfo, err := os.Stat(path) + if err != nil { + logger.Error("Error checking config file", "err", err) + continue + } + + modTime := fileInfo.ModTime() + if modTime.After(lastModTime) { + logger.Info("config file modified, reloading...") + lastModTime = modTime + + newConfig, err := parseConfigFile(path) + if err != nil { + logger.Error("failed to reload config", "err", err) + } else { + onChange(newConfig) + } + } + } +} + +func ensureConfig(repoPool *mirror.RepoPool, newConfig *mirror.RepoPoolConfig) { + + // add default values + applyGitDefaults(newConfig) + + // validate and apply defaults to new config before compare + if err := newConfig.ValidateAndApplyDefaults(); err != nil { + logger.Error("failed to validate new config", "err", err) + return + } + + newRepos, removedRepos := diffRepositories(repoPool, newConfig) + for _, repo := range removedRepos { + if err := repoPool.RemoveRepository(repo); err != nil { + logger.Error("failed to remove repository", "remote", repo, "err", err) + } + } + for _, repo := range newRepos { + if err := repoPool.AddRepository(repo); err != nil { + logger.Error("failed to add new repository", "remote", repo.Remote, "err", err) + } + } + + // find matched repos and check for worktree diffs + for _, newRepoConf := range newConfig.Repositories { + repo, err := repoPool.Repository(newRepoConf.Remote) + if err != nil { + continue + } + + newWTs, removedWTs := diffWorktrees(repo, &newRepoConf) + + // 1st remove then add new in case new one has same link with diff reference + for _, wt := range removedWTs { + if err := repoPool.RemoveWorktreeLink(newRepoConf.Remote, wt); err != nil { + logger.Error("failed to remove worktree", "remote", newRepoConf.Remote, "link", wt, "err", err) + } + } + for _, wt := range newWTs { + if err := repoPool.AddWorktreeLink(newRepoConf.Remote, wt); err != nil { + logger.Error("failed to add worktree", "remote", newRepoConf.Remote, "link", wt.Link, "err", err) + } + } + } +} + +func applyGitDefaults(mirrorConf *mirror.RepoPoolConfig) { + if mirrorConf.Defaults.Root == "" { + mirrorConf.Defaults.Root = defaultRoot + } + + if mirrorConf.Defaults.GitGC == "" { + mirrorConf.Defaults.GitGC = defaultGitGC + } + + if mirrorConf.Defaults.Interval == 0 { + mirrorConf.Defaults.Interval = defaultInterval + } + + if mirrorConf.Defaults.MirrorTimeout == 0 { + mirrorConf.Defaults.MirrorTimeout = defaultMirrorTimeout + } + + if mirrorConf.Defaults.Auth.SSHKeyPath == "" { + mirrorConf.Defaults.Auth.SSHKeyPath = defaultSSHKeyPath + } + + if mirrorConf.Defaults.Auth.SSHKnownHostsPath == "" { + mirrorConf.Defaults.Auth.SSHKnownHostsPath = defaultSSHKnownHostsPath + } +} + +func parseConfigFile(path string) (*mirror.RepoPoolConfig, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + return nil, err + } + conf := &mirror.RepoPoolConfig{} + err = yaml.Unmarshal(yamlFile, conf) + if err != nil { + return nil, err + } + return conf, nil +} + +func diffRepositories(repoPool *mirror.RepoPool, newConfig *mirror.RepoPoolConfig) ( + newRepos []mirror.RepositoryConfig, + removedRepos []string, +) { + for _, newRepo := range newConfig.Repositories { + if _, err := repoPool.Repository(newRepo.Remote); errors.Is(err, mirror.ErrNotExist) { + newRepos = append(newRepos, newRepo) + } + } + + for _, currentRepoURL := range repoPool.RepositoriesRemote() { + var found bool + for _, newRepo := range newConfig.Repositories { + if currentRepoURL == giturl.NormaliseURL(newRepo.Remote) { + found = true + break + } + } + if !found { + removedRepos = append(removedRepos, currentRepoURL) + } + } + + return +} + +func diffWorktrees(repo *mirror.Repository, newRepoConf *mirror.RepositoryConfig) ( + newWTCs []mirror.WorktreeConfig, + removedWTs []string, +) { + currentWTLinks := repo.WorktreeLinks() + + for _, newWTC := range newRepoConf.Worktrees { + if _, ok := currentWTLinks[newWTC.Link]; !ok { + newWTCs = append(newWTCs, newWTC) + } + } + + // for existing worktree + for cLink, wt := range currentWTLinks { + var found bool + for _, newWTC := range newRepoConf.Worktrees { + if newWTC.Link == cLink { + // wt link name is matching so make sure other + // config match as well if not replace it + if !wt.Equals(newWTC) { + newWTCs = append(newWTCs, newWTC) + break + } + found = true + break + } + } + if !found { + removedWTs = append(removedWTs, cLink) + } + } + + return +} diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000..e0c3623 --- /dev/null +++ b/config_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "log/slog" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/utilitywarehouse/git-mirror/pkg/mirror" +) + +func Test_diffRepositories(t *testing.T) { + + tests := []struct { + name string + initialConfig *mirror.RepoPoolConfig + newConfig *mirror.RepoPoolConfig + wantNewRepos []mirror.RepositoryConfig + wantRemovedRepos []string + }{ + { + name: "empty", + initialConfig: &mirror.RepoPoolConfig{}, + newConfig: &mirror.RepoPoolConfig{ + Defaults: mirror.DefaultConfig{Root: "/root"}, + Repositories: []mirror.RepositoryConfig{ + {Remote: "user@host.xz:path/to/repo1.git"}, + {Remote: "user@host.xz:path/to/repo2.git"}, + }, + }, + wantNewRepos: []mirror.RepositoryConfig{ + {Remote: "user@host.xz:path/to/repo1.git"}, + {Remote: "user@host.xz:path/to/repo2.git"}, + }, + wantRemovedRepos: nil, + }, + { + name: "replace_repo2_repo3", + initialConfig: &mirror.RepoPoolConfig{ + Defaults: mirror.DefaultConfig{Root: "/root", Interval: 10 * time.Second}, + Repositories: []mirror.RepositoryConfig{ + {Remote: "user@host.xz:path/to/repo1.git"}, + {Remote: "user@host.xz:path/to/repo2.git"}, + }, + }, + newConfig: &mirror.RepoPoolConfig{ + Defaults: mirror.DefaultConfig{Root: "/root"}, + Repositories: []mirror.RepositoryConfig{ + {Remote: "user@host.xz:path/to/repo1.git"}, + { + Remote: "user@host.xz:path/to/repo3.git", + Root: "/another-root", + Interval: 2 * time.Second, + MirrorTimeout: 4 * time.Second, + GitGC: "off", + Auth: mirror.Auth{SSHKeyPath: "/another/path/to/key"}, + }, + }, + }, + wantNewRepos: []mirror.RepositoryConfig{ + { + Remote: "user@host.xz:path/to/repo3.git", + Root: "/another-root", + Interval: 2 * time.Second, + MirrorTimeout: 4 * time.Second, + GitGC: "off", + Auth: mirror.Auth{SSHKeyPath: "/another/path/to/key"}, + }, + }, + wantRemovedRepos: []string{"user@host.xz:path/to/repo2.git"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + applyGitDefaults(tt.initialConfig) + repoPool, err := mirror.NewRepoPool(*tt.initialConfig, nil, nil) + if err != nil { + t.Fatalf("could not create git mirror pool err:%v", err) + } + + gotNewRepos, gotRemovedRepos := diffRepositories(repoPool, tt.newConfig) + if diff := cmp.Diff(gotNewRepos, tt.wantNewRepos); diff != "" { + t.Errorf("diffRepositories() NewRepos mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(gotRemovedRepos, tt.wantRemovedRepos); diff != "" { + t.Errorf("diffRepositories() RemovedRepos mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func Test_diffWorktrees(t *testing.T) { + tests := []struct { + name string + initialRepoConf *mirror.RepositoryConfig + newRepoConf *mirror.RepositoryConfig + wantNewWTCs []mirror.WorktreeConfig + wantRemovedWTs []string + }{ + { + name: "no_worktree", + initialRepoConf: &mirror.RepositoryConfig{ + Remote: "user@host.xz:path/to/repo1.git", + Root: "/root", Interval: 10 * time.Second, GitGC: "always", + }, + newRepoConf: &mirror.RepositoryConfig{ + Remote: "user@host.xz:path/to/repo1.git", + Root: "/root", Interval: 10 * time.Second, GitGC: "always", + Worktrees: []mirror.WorktreeConfig{ + {Link: "link", Ref: "master", Pathspec: ""}, + {Link: "link2", Ref: "other-branch", Pathspec: "path"}, + }, + }, + wantNewWTCs: []mirror.WorktreeConfig{ + {Link: "link", Ref: "master"}, + {Link: "link2", Ref: "other-branch", Pathspec: "path"}, + }, + wantRemovedWTs: nil, + }, + { + name: "replace_link_ref_path", + initialRepoConf: &mirror.RepositoryConfig{ + Remote: "user@host.xz:path/to/repo1.git", + Root: "/root", Interval: 10 * time.Second, GitGC: "always", + Worktrees: []mirror.WorktreeConfig{ + {Link: "link", Ref: "master", Pathspec: ""}, + {Link: "link2", Ref: "other-branch", Pathspec: "path"}, + }, + }, + newRepoConf: &mirror.RepositoryConfig{ + Remote: "user@host.xz:path/to/repo1.git", + Root: "/root", Interval: 10 * time.Second, GitGC: "always", + Worktrees: []mirror.WorktreeConfig{ + {Link: "link", Ref: "master", Pathspec: "new-path"}, + {Link: "link2", Ref: "new-branch", Pathspec: "path"}, + }, + }, + wantNewWTCs: []mirror.WorktreeConfig{ + {Link: "link", Ref: "master", Pathspec: "new-path"}, + {Link: "link2", Ref: "new-branch", Pathspec: "path"}, + }, + wantRemovedWTs: []string{"link", "link2"}, + }, + { + name: "add_new_link", + initialRepoConf: &mirror.RepositoryConfig{ + Remote: "user@host.xz:path/to/repo1.git", + Root: "/root", Interval: 10 * time.Second, GitGC: "always", + Worktrees: []mirror.WorktreeConfig{ + {Link: "link", Ref: "master", Pathspec: ""}, + {Link: "link2", Ref: "other-branch", Pathspec: "path"}, + }, + }, + newRepoConf: &mirror.RepositoryConfig{ + Remote: "user@host.xz:path/to/repo1.git", + Root: "/root", Interval: 10 * time.Second, GitGC: "always", + Worktrees: []mirror.WorktreeConfig{ + {Link: "link", Ref: "master", Pathspec: ""}, + {Link: "link3", Ref: "other-branch", Pathspec: "path"}, + }, + }, + wantNewWTCs: []mirror.WorktreeConfig{ + {Link: "link3", Ref: "other-branch", Pathspec: "path"}, + }, + wantRemovedWTs: []string{"link2"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + repo, err := mirror.NewRepository(*tt.initialRepoConf, nil, slog.Default()) + if err != nil { + t.Fatalf("failed to create repo error = %v", err) + } + + gotNewWTCs, gotRemovedWTs := diffWorktrees(repo, tt.newRepoConf) + + if diff := cmp.Diff(gotNewWTCs, tt.wantNewWTCs); diff != "" { + t.Errorf("diffWorktrees() NewWTCs mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(gotRemovedWTs, tt.wantRemovedWTs); diff != "" { + t.Errorf("diffWorktrees() RemovedWTs mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/main.go b/main.go index f3eb790..a948d8c 100644 --- a/main.go +++ b/main.go @@ -6,14 +6,12 @@ import ( "log/slog" "os" "os/signal" - "path" "strings" "syscall" "time" "github.com/urfave/cli/v3" "github.com/utilitywarehouse/git-mirror/pkg/mirror" - "gopkg.in/yaml.v3" ) var ( @@ -28,8 +26,6 @@ var ( "error": slog.LevelError, } - reposRootPath = path.Join(os.TempDir(), "git-mirror", "src") - flags = []cli.Flag{ &cli.StringFlag{ Name: "config", @@ -43,6 +39,12 @@ var ( Value: "info", Usage: "Log level", }, + &cli.BoolFlag{ + Name: "watch-config", + Usage: "watch config for changes and reload when changes encountered.\n" + + "Only changes related to add,remove repository or worktrees will be actioned.", + Value: true, + }, } ) @@ -53,77 +55,48 @@ func init() { })) } -func parseConfigFile(path string) (*mirror.RepoPoolConfig, error) { - yamlFile, err := os.ReadFile(path) - if err != nil { - return nil, err - } - conf := &mirror.RepoPoolConfig{} - err = yaml.Unmarshal(yamlFile, conf) - if err != nil { - return nil, err - } - return conf, nil -} - -func applyGitDefaults(c *cli.Command, mirrorConf *mirror.RepoPoolConfig) *mirror.RepoPoolConfig { - if mirrorConf.Defaults.Root == "" { - mirrorConf.Defaults.Root = reposRootPath - } - - if mirrorConf.Defaults.GitGC == "" { - mirrorConf.Defaults.GitGC = "always" - } - - if mirrorConf.Defaults.Interval == 0 { - mirrorConf.Defaults.Interval = 30 * time.Second - } - - if mirrorConf.Defaults.MirrorTimeout == 0 { - mirrorConf.Defaults.MirrorTimeout = 2 * time.Minute - } - - return mirrorConf -} - func main() { cmd := &cli.Command{ Name: "git-mirror", Usage: "git-mirror is a tool to periodically mirror remote repositories locally.", Flags: flags, Action: func(ctx context.Context, c *cli.Command) error { + ctx, cancel := context.WithCancel(ctx) // set log level according to argument if v, ok := levelStrings[strings.ToLower(c.String("log-level"))]; ok { loggerLevel.Set(v) } - conf, err := parseConfigFile(c.String("config")) - if err != nil { - logger.Error("unable to parse tf applier config file", "err", err) - os.Exit(1) - } - - // setup git-mirror - conf = applyGitDefaults(c, conf) - - // path to resolve strongbox + // path to resolve git gitENV := []string{fmt.Sprintf("PATH=%s", os.Getenv("PATH"))} - repos, err := mirror.NewRepoPool(*conf, logger.With("logger", "git-mirror"), gitENV) + // create empty repo pool which will be populated by watchConfig + repoPool, err := mirror.NewRepoPool(mirror.RepoPoolConfig{}, logger.With("logger", "git-mirror"), gitENV) if err != nil { logger.Error("could not create git mirror pool", "err", err) os.Exit(1) } - // start mirror Loop - repos.StartLoop() + onConfigChange := func(config *mirror.RepoPoolConfig) { + ensureConfig(repoPool, config) + // start mirror Loop on newly added repos + repoPool.StartLoop(ctx) + } + + // Start watching the config file + go WatchConfig(c.String("config"), 10*time.Second, onConfigChange) //listenForShutdown stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop - logger.Info("Shutting down") + + logger.Info("Shutting down...") + cancel() + + // TODO: wait for all repo sync to terminate return nil }, diff --git a/pkg/mirror/worktree.go b/pkg/mirror/worktree.go index 904d95a..0ea420e 100644 --- a/pkg/mirror/worktree.go +++ b/pkg/mirror/worktree.go @@ -16,6 +16,12 @@ type WorkTreeLink struct { log *slog.Logger } +func (wt *WorkTreeLink) Equals(wtc WorktreeConfig) bool { + return wt.link == wtc.Link && + wt.pathspec == wtc.Pathspec && + wt.ref == wtc.Ref +} + // worktreeDirName will generate worktree name for specific worktree link // two worktree links can be on same ref but with diff pathspecs // hence we cant just use tree hash as path From 023e4eb0b9c573bb0ac66f8ec45e025d8ae546c4 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 16:05:59 +0000 Subject: [PATCH 08/12] updated go and dependencies --- go.mod | 12 ++++++------ go.sum | 10 ++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b82a7e9..86ccedc 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/utilitywarehouse/git-mirror -go 1.23.0 +go 1.24.0 require ( github.com/google/go-cmp v0.7.0 - github.com/prometheus/client_golang v1.21.0 + github.com/prometheus/client_golang v1.21.1 github.com/sasha-s/go-deadlock v0.3.5 github.com/urfave/cli/v3 v3.0.0-beta1 gopkg.in/yaml.v3 v3.0.1 @@ -15,10 +15,10 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect + github.com/petermattis/goid v0.0.0-20250303134427-723919f7f203 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/common v0.63.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - golang.org/x/sys v0.28.0 // indirect - google.golang.org/protobuf v1.36.1 // indirect + golang.org/x/sys v0.31.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect ) diff --git a/go.sum b/go.sum index ec60408..0b60c85 100644 --- a/go.sum +++ b/go.sum @@ -17,14 +17,20 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/petermattis/goid v0.0.0-20250303134427-723919f7f203 h1:E7Kmf11E4K7B5hDti2K2NqPb1nlYlGYsu02S1JNd/Bs= +github.com/petermattis/goid v0.0.0-20250303134427-723919f7f203/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= +github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= +github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -37,8 +43,12 @@ github.com/urfave/cli/v3 v3.0.0-beta1 h1:6DTaaUarcM0wX7qj5Hcvs+5Dm3dyUTBbEwIWAjc github.com/urfave/cli/v3 v3.0.0-beta1/go.mod h1:FnIeEMYu+ko8zP1F9Ypr3xkZMIDqW3DR92yUtY39q1Y= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 405b2ac46b84c01df956a2d3d458baef19986ee0 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 16:14:06 +0000 Subject: [PATCH 09/12] fix test --- pkg/mirror/repository_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mirror/repository_test.go b/pkg/mirror/repository_test.go index e26762f..c9338d5 100644 --- a/pkg/mirror/repository_test.go +++ b/pkg/mirror/repository_test.go @@ -158,7 +158,7 @@ func TestRepo_AddWorktreeLink(t *testing.T) { "link": {link: "link", linkAbs: "/tmp/root/link", ref: "master"}, "link2": {link: "link2", linkAbs: "/tmp/root/link2", ref: "other-branch", pathspec: "path"}, "link3": {link: "link3", linkAbs: "/tmp/root/link3", ref: "HEAD"}, - "/tmp/link": {link: "link", linkAbs: "/tmp/link", ref: "tag"}, + "/tmp/link": {link: "/tmp/link", linkAbs: "/tmp/link", ref: "tag"}, } if diff := cmp.Diff(want, r.workTreeLinks, cmpopts.IgnoreFields(WorkTreeLink{}, "log"), cmp.AllowUnexported(WorkTreeLink{})); diff != "" { t.Errorf("Repo.AddWorktreeLink() worktreelinks mismatch (-want +got):\n%s", diff) From f1d7640641c9eb54076ea9814bdc1d827b2fac23 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 16:17:10 +0000 Subject: [PATCH 10/12] update test version --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c62c270..5466b22 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,7 @@ jobs: pkg-test: strategy: matrix: - go-version: [1.21.x, 1.22.x, 1.23.x] + go-version: [1.23.x, 1.24.x] os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: From 3d7b7cf794a379113d4d361dd76b5acb52620f83 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 16:20:55 +0000 Subject: [PATCH 11/12] fix race detection test --- pkg/mirror/z_e2e_race_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mirror/z_e2e_race_test.go b/pkg/mirror/z_e2e_race_test.go index 687304a..f408027 100644 --- a/pkg/mirror/z_e2e_race_test.go +++ b/pkg/mirror/z_e2e_race_test.go @@ -31,7 +31,7 @@ func Test_mirror_detect_race(t *testing.T) { repo := mustCreateRepoAndMirror(t, upstream, root, link1, ref1) // add worktree for HEAD - if err := repo.AddWorktreeLink(link2, ref2, ""); err != nil { + if err := repo.AddWorktreeLink(WorktreeConfig{link2, ref2, ""}); err != nil { t.Fatalf("unable to add worktree error: %v", err) } // mirror again for 2nd worktree From e64593fb208ce8c6fbbfa2a5d6f8175de52ae60e Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Mon, 17 Mar 2025 16:30:47 +0000 Subject: [PATCH 12/12] produce consistent test results --- config_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config_test.go b/config_test.go index e0c3623..972901a 100644 --- a/config_test.go +++ b/config_test.go @@ -2,6 +2,7 @@ package main import ( "log/slog" + "slices" "testing" "time" @@ -175,6 +176,20 @@ func Test_diffWorktrees(t *testing.T) { gotNewWTCs, gotRemovedWTs := diffWorktrees(repo, tt.newRepoConf) + // since these slices are based on map of worktrees order of elements + // differs between runs + slices.SortFunc(gotNewWTCs, func(a, b mirror.WorktreeConfig) int { + switch { + case a.Link > b.Link: + return 1 + case a.Link == b.Link: + return 0 + default: + return -1 + } + }) + slices.Sort(gotRemovedWTs) + if diff := cmp.Diff(gotNewWTCs, tt.wantNewWTCs); diff != "" { t.Errorf("diffWorktrees() NewWTCs mismatch (-want +got):\n%s", diff) }