-
Notifications
You must be signed in to change notification settings - Fork 81
/
sync.go
160 lines (140 loc) · 4.15 KB
/
sync.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
package helm
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"helm.sh/helm/v3/pkg/repo"
helm_repo "helm.sh/helm/v3/pkg/repo"
"kubegems.io/kubegems/pkg/log"
)
// MaxSyncVerionCount 最大同步版本,仅同步某个chart的最近的 n 个版本
const MaxSyncVerionCount = 3
type ProcessEvent struct {
Chart *repo.ChartVersion
Error error
Message string
}
var syncState sync.Map
var ErrSynchronizing = errors.New("synchronizing")
func SyncChartsToChartmuseum(ctx context.Context, remote RepositoryConfig, localChartMuseum RepositoryConfig) error {
onProcess := func(e ProcessEvent) {
if e.Error != nil {
log.Errorf("sync chart %s:%s, error: %s", e.Chart.Name, e.Chart.Version, e.Error.Error())
} else {
log.Infof("sync chart %s:%s,message: %s", e.Chart.Name, e.Chart.Version, e.Message)
}
}
return SyncChartsToChartmuseumWithProcess(ctx, remote, localChartMuseum, onProcess)
}
func SyncChartsToChartmuseumWithProcess(ctx context.Context, remote RepositoryConfig, localChartMuseum RepositoryConfig, onProcess func(ProcessEvent)) error {
if onProcess == nil {
onProcess = func(ProcessEvent) {}
}
remoteRepository, err := NewLegencyRepository(&remote)
if err != nil {
return err
}
if remote.Name == "" {
return errors.New("empty remote repository name")
}
// state marchine
if _, ok := syncState.Load(remote.Name); ok {
return fmt.Errorf("repo %s %w", remote.Name, ErrSynchronizing)
}
syncState.Store(remote.Name, struct{}{})
defer syncState.Delete(remote.Name)
index, err := remoteRepository.GetIndex(ctx)
if err != nil {
return fmt.Errorf("download index: %w", err)
}
chartmuseum, err := NewChartMuseumClient(&localChartMuseum)
if err != nil {
return err
}
if err := chartmuseum.Health(ctx); err != nil {
return fmt.Errorf("chartmuseum health check failed: %w", err)
}
errors := []error{}
// 暂时使用 1 并发
concurrent := NewGoconcurrent(1)
// get local repo index
localIndex, err := chartmuseum.ListAllChartVersions(ctx, remote.Name)
if err != nil {
log.Warnf("chartmuseum repo %s inedx error: %s", err.Error())
}
findLocalVersion := func(name, version string) *helm_repo.ChartVersion {
if cvs, ok := localIndex[name]; ok {
for _, v := range cvs {
if v.Version == version {
return v
}
}
}
return nil
}
for i := range index.Entries {
versions := index.Entries[i]
for j := range versions {
// j start from 0 ,using >=
if j >= MaxSyncVerionCount {
break
}
version := versions[j]
concurrent.Go(func() {
onProcess(ProcessEvent{Chart: version, Message: "checking"})
if len(version.URLs) == 0 {
err := fmt.Errorf("chart version %s no urls found", version.Name)
errors = append(errors, err)
onProcess(ProcessEvent{Chart: version, Error: err})
return
}
// 检查Digest,若相同则跳过
if existVersion := findLocalVersion(version.Name, version.Version); existVersion != nil && existVersion.Digest == version.Digest {
onProcess(ProcessEvent{Chart: version, Message: "already exist and uptodate"})
return
}
onProcess(ProcessEvent{Chart: version, Message: "downloading"})
readercloser, err := remoteRepository.GetFile(ctx, version.URLs[0])
if err != nil {
errors = append(errors, err)
onProcess(ProcessEvent{Chart: version, Error: err})
return
}
onProcess(ProcessEvent{Chart: version, Message: "pushing"})
if err := chartmuseum.UploadChart(ctx, remote.Name, readercloser); err != nil {
errors = append(errors, err)
onProcess(ProcessEvent{Chart: version, Error: err})
return
}
readercloser.Close()
onProcess(ProcessEvent{Chart: version, Message: "synced"})
})
}
}
if len(errors) > 0 {
return Errors(errors)
}
return nil
}
type Errors []error
func (errs Errors) Error() string {
builder := strings.Builder{}
for _, err := range errs {
builder.WriteString(err.Error())
builder.WriteString(";")
}
return builder.String()
}
func NewGoconcurrent(concurrent int) *Goconcurrent {
return &Goconcurrent{state: make(chan struct{}, concurrent)}
}
type Goconcurrent struct {
state chan struct{}
}
func (g *Goconcurrent) Go(f func()) {
g.state <- struct{}{}
f()
<-g.state
}