Skip to content

Commit 926e790

Browse files
ayu-devtronayu-devtron
committed
reload materials handling + refactoring
1 parent 9033236 commit 926e790

File tree

15 files changed

+1079
-787
lines changed

15 files changed

+1079
-787
lines changed

‎api/GrpcHandler.go‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type GrpcHandler interface {
2727
RefreshGitMaterial(ctx context.Context, req *pb.RefreshGitMaterialRequest) (*pb.RefreshGitMaterialResponse, error)
2828
ReloadAllMaterial(ctx context.Context, req *pb.Empty) (*pb.Empty, error)
2929
ReloadMaterial(ctx context.Context, req *pb.ReloadMaterialRequest) (*pb.GenericResponse, error)
30+
ReloadMaterials(ctx context.Context, req *pb.ReloadMaterialsRequest) (*pb.GenericResponse, error)
3031
GetChangesInRelease(ctx context.Context, req *pb.ReleaseChangeRequest) (*pb.GitChanges, error)
3132
GetWebhookData(ctx context.Context, req *pb.WebhookDataRequest) (*pb.WebhookAndCiData, error)
3233
GetAllWebhookEventConfigForHost(ctx context.Context, req *pb.WebhookEventConfigRequest) (*pb.WebhookEventConfigResponse, error)
@@ -103,7 +104,11 @@ func (impl *GrpcHandlerImpl) AddRepo(ctx context.Context, req *pb.AddRepoRequest
103104
}
104105
}
105106

106-
gitCtx := git.BuildGitContext(ctx)
107+
cloningMode := git.CloningModeFull
108+
if req.GitMaterialList != nil || len(req.GitMaterialList) > 0 {
109+
cloningMode = req.GitMaterialList[0].GetCloningMode()
110+
}
111+
gitCtx := git.BuildGitContext(ctx).WithCloningMode(cloningMode)
107112

108113
_, err := impl.repositoryManager.AddRepo(gitCtx, gitMaterials)
109114
if err != nil {
@@ -117,7 +122,7 @@ func (impl *GrpcHandlerImpl) AddRepo(ctx context.Context, req *pb.AddRepoRequest
117122
// UpdateRepo updates GitMaterial
118123
func (impl *GrpcHandlerImpl) UpdateRepo(ctx context.Context, req *pb.GitMaterial) (
119124
*pb.Empty, error) {
120-
gitCtx := git.BuildGitContext(ctx)
125+
gitCtx := git.BuildGitContext(ctx).WithCloningMode(req.GetCloningMode())
121126

122127
// Mapping
123128
mappedGitMaterial := &sql.GitMaterial{
@@ -285,8 +290,6 @@ func (impl *GrpcHandlerImpl) GetHeadForPipelineMaterials(ctx context.Context, re
285290
func (impl *GrpcHandlerImpl) GetCommitMetadata(ctx context.Context, req *pb.CommitMetadataRequest) (
286291
*pb.GitCommit, error) {
287292

288-
gitCtx := git.BuildGitContext(ctx)
289-
290293
// Mapping req body
291294
mappedReq := &git.CommitMetadataRequest{
292295
PipelineMaterialId: int(req.PipelineMaterialId),
@@ -297,6 +300,7 @@ func (impl *GrpcHandlerImpl) GetCommitMetadata(ctx context.Context, req *pb.Comm
297300

298301
var gitCommit *git.GitCommitBase
299302
var err error
303+
gitCtx := git.BuildGitContext(ctx)
300304

301305
if len(req.GitTag) > 0 {
302306
gitCommit, err = impl.repositoryManager.GetCommitInfoForTag(gitCtx, mappedReq)
@@ -464,6 +468,24 @@ func (impl *GrpcHandlerImpl) ReloadMaterial(ctx context.Context, req *pb.ReloadM
464468
}, nil
465469
}
466470

471+
func (impl *GrpcHandlerImpl) ReloadMaterials(ctx context.Context, req *pb.ReloadMaterialsRequest) (
472+
*pb.GenericResponse, error) {
473+
for _, material := range req.GetReloadMaterials() {
474+
gitCtx := git.BuildGitContext(ctx).WithCloningMode(material.GetCloningMode())
475+
err := impl.repositoryManager.ResetRepo(gitCtx, int(material.MaterialId))
476+
if err != nil {
477+
impl.logger.Errorw("error while reloading material",
478+
"materialId", material.MaterialId,
479+
"err", err)
480+
481+
}
482+
}
483+
484+
return &pb.GenericResponse{
485+
Message: "reloaded",
486+
}, nil
487+
}
488+
467489
func (impl *GrpcHandlerImpl) GetChangesInRelease(ctx context.Context, req *pb.ReleaseChangeRequest) (
468490
*pb.GitChanges, error) {
469491

‎api/ReloadMaterialsDto.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package api
2+
3+
type ReloadMaterialsDto struct {
4+
reloadMaterial []ReloadMaterialDto
5+
}
6+
7+
type ReloadMaterialDto struct {
8+
AppId int `json:"appId"`
9+
GitmaterialId int `json:"gitmaterialId"`
10+
CloningMode string `json:"cloningMode"`
11+
}

‎api/RestHandler.go‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ type RestHandler interface {
3838
GetCommitMetadataForPipelineMaterial(w http.ResponseWriter, r *http.Request)
3939
ReloadAllMaterial(w http.ResponseWriter, r *http.Request)
4040
ReloadMaterial(w http.ResponseWriter, r *http.Request)
41+
ReloadMaterials(w http.ResponseWriter, r *http.Request)
4142
GetChangesInRelease(w http.ResponseWriter, r *http.Request)
4243
GetCommitInfoForTag(w http.ResponseWriter, r *http.Request)
4344
RefreshGitMaterial(w http.ResponseWriter, r *http.Request)
@@ -116,9 +117,13 @@ func (handler RestHandlerImpl) SaveGitProvider(w http.ResponseWriter, r *http.Re
116117
func (handler RestHandlerImpl) AddRepo(w http.ResponseWriter, r *http.Request) {
117118
decoder := json.NewDecoder(r.Body)
118119

119-
gitCtx := git.BuildGitContext(r.Context())
120120
var Repo []*sql.GitMaterial
121121
err := decoder.Decode(&Repo)
122+
cloningMode := git.CloningModeFull
123+
if Repo != nil && len(Repo) > 0 {
124+
cloningMode = Repo[0].CloningMode
125+
}
126+
gitCtx := git.BuildGitContext(r.Context()).WithCloningMode(cloningMode)
122127
if err != nil {
123128
handler.logger.Error(err)
124129
handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
@@ -135,9 +140,9 @@ func (handler RestHandlerImpl) AddRepo(w http.ResponseWriter, r *http.Request) {
135140

136141
func (handler RestHandlerImpl) UpdateRepo(w http.ResponseWriter, r *http.Request) {
137142
decoder := json.NewDecoder(r.Body)
138-
gitCtx := git.BuildGitContext(r.Context())
139143
var Repo *sql.GitMaterial
140144
err := decoder.Decode(&Repo)
145+
gitCtx := git.BuildGitContext(r.Context()).WithCloningMode(Repo.CloningMode)
141146
if err != nil {
142147
handler.logger.Error(err)
143148
handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
@@ -180,6 +185,31 @@ func (handler RestHandlerImpl) ReloadAllMaterial(w http.ResponseWriter, r *http.
180185
handler.writeJsonResp(w, nil, "reloaded se logs for detail", http.StatusOK)
181186
}
182187

188+
func (handler RestHandlerImpl) ReloadMaterials(w http.ResponseWriter, r *http.Request) {
189+
decoder := json.NewDecoder(r.Body)
190+
var request ReloadMaterialsDto
191+
err := decoder.Decode(&request)
192+
193+
//materialId, err := strconv.Atoi(vars["materialId"])
194+
if err != nil {
195+
handler.logger.Error(err)
196+
handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
197+
return
198+
}
199+
for _, materialReq := range request.reloadMaterial {
200+
handler.logger.Infow("reload all pipelineMaterial request", "id", materialReq.GitmaterialId)
201+
gitCtx := git.BuildGitContext(r.Context()).WithCloningMode(materialReq.CloningMode)
202+
err = handler.repositoryManager.ResetRepo(gitCtx, materialReq.GitmaterialId)
203+
if err != nil {
204+
handler.logger.Errorw("error in reloading pipeline material", "err", err)
205+
//handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
206+
}
207+
}
208+
//TODO: handle in such a way that it can propagate which material weren't able to reload
209+
handler.writeJsonResp(w, nil, "reloaded", http.StatusOK)
210+
211+
}
212+
183213
func (handler RestHandlerImpl) ReloadMaterial(w http.ResponseWriter, r *http.Request) {
184214
vars := mux.Vars(r)
185215
gitCtx := git.BuildGitContext(r.Context())
@@ -238,7 +268,6 @@ func (handler RestHandlerImpl) GetHeadForPipelineMaterials(w http.ResponseWriter
238268

239269
func (handler RestHandlerImpl) GetCommitMetadata(w http.ResponseWriter, r *http.Request) {
240270
decoder := json.NewDecoder(r.Body)
241-
gitCtx := git.BuildGitContext(r.Context())
242271

243272
material := &git.CommitMetadataRequest{}
244273
err := decoder.Decode(material)
@@ -247,6 +276,7 @@ func (handler RestHandlerImpl) GetCommitMetadata(w http.ResponseWriter, r *http.
247276
handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
248277
return
249278
}
279+
gitCtx := git.BuildGitContext(r.Context())
250280
handler.logger.Infow("commit detail request", "req", material)
251281
var commits *git.GitCommitBase
252282
if len(material.GitTag) > 0 {

‎api/Router.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ func (r MuxRouter) Init() {
7575

7676
r.Router.Path("/admin/reload-all").HandlerFunc(r.restHandler.ReloadAllMaterial).Methods("POST")
7777
r.Router.Path("/admin/reload/{materialId}").HandlerFunc(r.restHandler.ReloadMaterial).Methods("POST")
78+
r.Router.Path("/admin/reload/materials").HandlerFunc(r.restHandler.ReloadMaterials).Methods("POST")
7879

7980
r.Router.Path("/release/changes").HandlerFunc(r.restHandler.GetChangesInRelease).Methods("POST")
8081

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.20
55
require (
66
github.com/caarlos0/env v3.5.0+incompatible
77
github.com/devtron-labs/common-lib v0.0.16-0.20240503082245-7ab933f8381f
8-
github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb
8+
github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1
99
github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab
1010
github.com/go-git/go-git/v5 v5.11.0
1111
github.com/go-pg/pg v6.15.1+incompatible

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
2929
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3030
github.com/devtron-labs/common-lib v0.0.16-0.20240503082245-7ab933f8381f h1:DcnCVtWaqwL6rptUD2eMaSwQEIL4/VaRWBfjyJbXy4U=
3131
github.com/devtron-labs/common-lib v0.0.16-0.20240503082245-7ab933f8381f/go.mod h1:mzEk3pf4JDXsOPNvCxjoSnsMBTwmt4A73Qc2tobLkZM=
32-
github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb h1:CkfQQgZc950/hTPqtQSiHV2RmZgkBLGCzwR02FZYjAU=
33-
github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb/go.mod h1:pjLjgoa1GzbkOkvbMyP4SAKsaiK7eG6GoQCNauG03JA=
32+
github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1 h1:R6qVeFaayqstBSu4w+ipWQqJyMKDqBVV3a11qoA2IaM=
33+
github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
3434
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
3535
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
3636
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=

‎internals/Configuration.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ type Configuration struct {
66
CommitStatsTimeoutInSec int `env:"COMMIT_STATS_TIMEOUT_IN_SEC" envDefault:"2"`
77
EnableFileStats bool `env:"ENABLE_FILE_STATS" envDefault:"false"`
88
GitHistoryCount int `env:"GIT_HISTORY_COUNT" envDefault:"15"`
9-
CloningMode string `env:"CLONING_MODE" envDefault:"FULL"`
109
MinLimit int `env:"MIN_LIMIT_FOR_PVC" envDefault:"1"` // in MB
1110
UseGitCli bool `env:"USE_GIT_CLI" envDefault:"false"`
1211
AnalyticsDebug bool `env:"ANALYTICS_DEBUG" envDefault:"false"`

‎internals/sql/GitMaterial.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ const (
3131
SOURCE_TYPE_WEBHOOK SourceType = "WEBHOOK"
3232
)
3333

34-
//TODO: add support for submodule
34+
// TODO: add support for submodule
3535
type GitMaterial struct {
3636
tableName struct{} `sql:"git_material"`
3737
Id int `sql:"id,pk"`
@@ -48,6 +48,7 @@ type GitMaterial struct {
4848
FetchStatus bool `json:"fetch_status"`
4949
LastFetchErrorCount int `json:"last_fetch_error_count"` //continues fetch error
5050
FetchErrorMessage string `json:"fetch_error_message"`
51+
CloningMode string `json:"cloning_mode"`
5152
FilterPattern []string `sql:"filter_pattern"`
5253
GitProvider *GitProvider
5354
CiPipelineMaterials []*CiPipelineMaterial

‎pkg/RepoManages.go‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,7 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
195195
continue
196196
}
197197

198-
gitCtx = gitCtx.WithCredentials(material.GitProvider.UserName, material.GitProvider.Password).
199-
WithCloningMode(impl.configuration.CloningMode)
198+
gitCtx = gitCtx.WithCredentials(material.GitProvider.UserName, material.GitProvider.Password)
200199

201200
fetchCount := impl.configuration.GitHistoryCount
202201
var repository *git.GitRepository
@@ -351,8 +350,7 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
351350
return material, nil
352351
}
353352

354-
gitCtx = gitCtx.WithCredentials(userName, password).
355-
WithCloningMode(impl.configuration.CloningMode)
353+
gitCtx = gitCtx.WithCredentials(userName, password)
356354

357355
checkoutPath, _, _, err := impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode)
358356
if err != nil {
@@ -638,8 +636,7 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe
638636

639637
userName, password, err := git.GetUserNamePassword(gitMaterial.GitProvider)
640638

641-
gitCtx = gitCtx.WithCredentials(userName, password).
642-
WithCloningMode(impl.configuration.CloningMode)
639+
gitCtx = gitCtx.WithCredentials(userName, password)
643640

644641
updated, repo, err := impl.repositoryManager.Fetch(gitCtx, gitMaterial.Url, gitMaterial.CheckoutLocation)
645642
if !updated {
@@ -707,8 +704,7 @@ func (impl RepoManagerImpl) GetCommitMetadataForPipelineMaterial(gitCtx git.GitC
707704
return nil, err
708705
}
709706

710-
gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password).
711-
WithCloningMode(impl.configuration.CloningMode)
707+
gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password)
712708
// validate checkout status of gitMaterial
713709
if !gitMaterial.CheckoutStatus {
714710
impl.logger.Errorw("checkout not success", "gitMaterialId", gitMaterialId)
@@ -763,8 +759,7 @@ func (impl RepoManagerImpl) GetReleaseChanges(gitCtx git.GitContext, request *Re
763759
impl.locker.ReturnLocker(gitMaterial.Id)
764760
}()
765761

766-
gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password).
767-
WithCloningMode(impl.configuration.CloningMode)
762+
gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password)
768763

769764
gitChanges, err := impl.repositoryManagerAnalytics.ChangesSinceByRepositoryForAnalytics(gitCtx, gitMaterial.CheckoutLocation, request.OldCommit, request.NewCommit)
770765
if err != nil {

‎pkg/git/GitContextUtils.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ func (gitCtx GitContext) WithTimeout(timeoutSeconds int) (GitContext, context.Ca
3131
}
3232

3333
func (gitCtx GitContext) WithCloningMode(CloningMode string) GitContext {
34+
if CloningMode == "" {
35+
CloningMode = CloningModeFull
36+
}
3437
gitCtx.CloningMode = CloningMode
3538
return gitCtx
3639
}

0 commit comments

Comments
 (0)