/
clone.go
176 lines (153 loc) · 6.2 KB
/
clone.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
package cmd
import (
"fmt"
"path/filepath"
"time"
"github.com/cidverse/go-vcsapp/pkg/platform/api"
"github.com/cidverse/go-vcsapp/pkg/vcsapp"
"github.com/cidverse/reposync/pkg/clone"
"github.com/cidverse/reposync/pkg/config"
"github.com/cidverse/reposync/pkg/repository"
"github.com/cidverse/reposync/pkg/util"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
)
func cloneCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "clone",
Aliases: []string{},
Short: `clones all repositories from configured remote servers`,
Run: func(cmd *cobra.Command, args []string) {
// flags
dryRun, err := cmd.Flags().GetBool("dry-run")
if err != nil {
log.Fatal().Err(err).Msg("failed to parse dry-run flag")
}
silent, err := cmd.Flags().GetBool("silent")
if err != nil {
log.Fatal().Err(err).Msg("failed to parse silent flag")
}
// config
c, err := config.Load()
if err != nil {
log.Fatal().Err(err).Str("file", cfg.ConfigFile).Msg("failed to parse config file")
}
// state
stateFile := config.StateFile()
state, err := config.LoadState(stateFile)
if err != nil {
log.Fatal().Err(err).Str("file", cfg.ConfigFile).Msg("failed to parse state file")
}
defer func(state *config.SyncState) { // ensure state is updated on exit
saveErr := config.SaveState(stateFile, state)
if saveErr != nil {
log.Fatal().Err(saveErr).Msg("failed to save state")
}
}(state)
// servers
for _, s := range c.Servers {
// skip if no local dir is specified
if s.Mirror.LocalDir == "" {
log.Debug().Str("server", s.Server).Str("type", s.Type).Msg("no local dir specified, skipping")
continue
}
// setup platform
log.Info().Str("server", s.Server).Str("type", s.Type).Msg("querying server")
platformAuth, platformAuthErr := config.AuthToPlatformConfig(s.Type, s.Server, s.Auth)
if platformAuthErr != nil {
log.Error().Err(platformAuthErr).Msg("failed to initialize platform auth")
continue
}
platform, platformErr := vcsapp.NewPlatform(platformAuth)
if platformErr != nil {
log.Error().Err(platformErr).Msg("failed to initialize platform")
continue
}
// query repositories
repos, repoErr := platform.Repositories(api.RepositoryListOpts{IncludeBranches: false, IncludeCommitHash: false})
if repoErr != nil {
log.Fatal().Err(repoErr).Msg("failed to list repositories")
}
log.Info().Int("count", len(repos)).Str("server", s.Server).Msg("received repository list")
// process repositories
for _, r := range repos {
uniqueId := fmt.Sprintf("%s/%d", r.PlatformId, r.Id)
log.Info().Str("namespace", r.Namespace).Str("repo", r.Name).Msg("processing repository")
// check rules
if config.EvaluateRules(s.Mirror.Rules, s.Mirror.DefaultAction, r) == config.RuleActionExclude {
log.Debug().Str("namespace", r.Namespace).Str("repo", r.Name).Msg("repository excluded by rules, skipping")
continue
}
// remote
remote := r.CloneURL
if s.Mirror.CloneMethod == config.CloneMethodSSH {
remote = r.CloneSSH
}
// expected state
expectedState := config.RepositoryState{
ID: uniqueId,
Namespace: r.Namespace,
Name: r.Name,
Remote: remote,
Directory: filepath.Join(util.ResolvePath(s.Mirror.LocalDir), util.Slugify(r.Namespace, string(s.Mirror.NamingStyle)), util.Slugify(r.Name, string(s.Mirror.NamingStyle))),
LastSync: time.Now(),
}
// current state
currentState, inCurrentState := state.Repositories[uniqueId]
// run actions based on state
if !inCurrentState {
log.Debug().Str("repo", r.Namespace+"/"+r.Name).Str("current-dir", currentState.Directory).Str("expected-dir", expectedState.Directory).Msg("repository not present, cloning")
if dryRun {
continue
}
// add untracked repository in expected location, add to state (can occur if clone is interrupted and the state is not updated)
if repository.Exists(expectedState.Directory) {
log.Info().Str("repo", r.Namespace+"/"+r.Name).Str("dir", expectedState.Directory).Msg("adding existing repository to state")
state.Repositories[uniqueId] = expectedState
continue
}
// clone repository
cloneErr := repository.CloneRepository(expectedState.Directory, remote, silent)
if cloneErr != nil {
log.Error().Err(cloneErr).Str("repo", r.Namespace+"/"+r.Name).Msg("failed to clone repository")
continue
}
} else if currentState.Directory != expectedState.Directory {
log.Debug().Str("repo", r.Namespace+"/"+r.Name).Str("current-dir", currentState.Directory).Str("expected-dir", expectedState.Directory).Msg("repository present in different location, moving")
if dryRun {
continue
}
// move repository
moveErr := repository.MoveRepository(currentState.Directory, expectedState.Directory)
if moveErr != nil {
log.Error().Err(moveErr).Str("repo", r.Namespace+"/"+r.Name).Str("current-dir", currentState.Directory).Str("expected-dir", expectedState.Directory).Msg("failed to move repository")
continue
}
} else if currentState.Directory == expectedState.Directory {
log.Debug().Str("repo", r.Namespace+"/"+r.Name).Str("dir", expectedState.Directory).Msg("repository already present in expected location")
}
// dry-run
if dryRun {
continue
}
// update remote (allows easy switching between https/ssh for all repositories)
updateRemoteErr := repository.UpdateRemote(expectedState.Directory, remote, silent)
if updateRemoteErr != nil {
log.Error().Err(updateRemoteErr).Str("repo", r.Namespace+"/"+r.Name).Msg("failed to update remote")
continue
}
// add to state
state.Repositories[uniqueId] = expectedState
}
}
// clone sources
for _, s := range c.Sources {
log.Debug().Str("remote", s.Url).Str("remote-ref", s.Ref).Str("target", s.TargetDir).Msg("processing project")
clone.FetchProject(s, s.TargetDir)
}
},
}
cmd.PersistentFlags().BoolP("dry-run", "d", false, "dry run")
cmd.PersistentFlags().BoolP("silent", "s", false, "silent (omit stdout/stderr output) from cli commands")
return cmd
}