fix: reload all API context handling and added range support for git material ids (#109)

* fix: reload all API context handling and added range support for git material ids

* vendor files for gorilla/schema

* fix: FindInRage query

Author: Ash-exp

api/GrpcHandler.go

@@ -464,8 +464,8 @@ func (impl *GrpcHandlerImpl) RefreshGitMaterial(ctx context.Context, req *pb.Ref
 
 func (impl *GrpcHandlerImpl) ReloadAllMaterial(ctx context.Context, req *pb.Empty) (*pb.Empty, error) {
 	gitCtx := git.BuildGitContext(ctx)
-	impl.repositoryManager.ReloadAllRepo(gitCtx)
-	return &pb.Empty{}, nil
+	err := impl.repositoryManager.ReloadAllRepo(gitCtx, nil)
+	return &pb.Empty{}, err
 }
 
 func (impl *GrpcHandlerImpl) ReloadMaterial(ctx context.Context, req *pb.ReloadMaterialRequest) (

api/RestHandler.go

@@ -18,10 +18,13 @@ package api
 
 import (
 	"encoding/json"
+	"fmt"
+	"github.com/devtron-labs/git-sensor/bean"
 	"github.com/devtron-labs/git-sensor/internals/sql"
 	"github.com/devtron-labs/git-sensor/pkg"
 	"github.com/devtron-labs/git-sensor/pkg/git"
 	"github.com/gorilla/mux"
+	"github.com/gorilla/schema"
 	"go.uber.org/zap"
 	"net/http"
 	"strconv"
@@ -179,10 +182,30 @@ func (handler RestHandlerImpl) SavePipelineMaterial(w http.ResponseWriter, r *ht
 }
 
 func (handler RestHandlerImpl) ReloadAllMaterial(w http.ResponseWriter, r *http.Request) {
-	handler.logger.Infow("reload all pipelineMaterial request")
+	handler.logger.Infow(bean.GetReloadAllLog("received reload all pipelineMaterial request"))
+	decoder := schema.NewDecoder()
+	decoder.IgnoreUnknownKeys(true)
+	req := &bean.ReloadAllMaterialQuery{}
+	err := decoder.Decode(req, r.URL.Query())
+	if err != nil {
+		handler.logger.Errorw(bean.GetReloadAllLog("invalid query params, ReloadAllMaterial"), "err", err, "query", r.URL.Query())
+		handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	if req.Start < 0 || req.End < 0 || req.Start > req.End {
+		handler.logger.Errorw(bean.GetReloadAllLog("invalid request, ReloadAllMaterial"), "query", r.URL.Query())
+		handler.writeJsonResp(w, fmt.Errorf("invalid query params"), nil, http.StatusBadRequest)
+		return
+	}
 	gitCtx := git.BuildGitContext(r.Context())
-	handler.repositoryManager.ReloadAllRepo(gitCtx)
-	handler.writeJsonResp(w, nil, "reloaded se logs for detail", http.StatusOK)
+	err = handler.repositoryManager.ReloadAllRepo(gitCtx, req)
+	if err != nil {
+		handler.logger.Errorw(bean.GetReloadAllLog("error in request, ReloadAllMaterial"), "query", r.URL.Query(), "err", err)
+		handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	handler.logger.Infow(bean.GetReloadAllLog("Reloaded materials successfully!"), "query", r.URL.Query())
+	handler.writeJsonResp(w, nil, "Reloaded materials successfully!", http.StatusOK)
 }
 
 func (handler RestHandlerImpl) ReloadMaterials(w http.ResponseWriter, r *http.Request) {

bean/bean.go

@@ -16,7 +16,20 @@
 
 package bean
 
+import "fmt"
+
 type StartupConfig struct {
 	RestPort int `env:"SERVER_REST_PORT" envDefault:"8080"`
 	GrpcPort int `env:"SERVER_GRPC_PORT" envDefault:"8081"`
 }
+
+const ReloadAllLogPrefix = "RELOAD_ALL_LOG"
+
+func GetReloadAllLog(message string) string {
+	return fmt.Sprintf("%s: %s", ReloadAllLogPrefix, message)
+}
+
+type ReloadAllMaterialQuery struct {
+	Start int `json:"start"`
+	End   int `json:"end"`
+}

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/google/wire v0.6.0
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/schema v1.1.0
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/prometheus/client_golang v1.16.0

go.sum

@@ -64,6 +64,8 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR
 github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY=
+github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY=

internals/sql/GitMaterial.go

@@ -60,6 +60,7 @@ type MaterialRepository interface {
 	Save(material *GitMaterial) error
 	FindActive() ([]*GitMaterial, error)
 	FindAll() ([]*GitMaterial, error)
+	FindInRage(startFrom int, endAt int) ([]*GitMaterial, error)
 	FindAllActiveByUrls(urls []string) ([]*GitMaterial, error)
 }
 type MaterialRepositoryImpl struct {
@@ -103,6 +104,21 @@ func (repo MaterialRepositoryImpl) FindAll() ([]*GitMaterial, error) {
 	return materials, err
 }
 
+func (repo MaterialRepositoryImpl) FindInRage(startFrom int, endAt int) ([]*GitMaterial, error) {
+	var materials []*GitMaterial
+	query := repo.dbConnection.Model(&materials).
+		Column("git_material.*", "GitProvider").
+		Where("deleted =? ", false)
+	if startFrom != 0 {
+		query.Where("git_material.id >= ?", startFrom)
+	}
+	if endAt != 0 {
+		query.Where("git_material.id <= ?", endAt)
+	}
+	err := query.Select()
+	return materials, err
+}
+
 func (repo MaterialRepositoryImpl) FindById(id int) (*GitMaterial, error) {
 	var material GitMaterial
 	err := repo.dbConnection.Model(&material).

pkg/RepoManages.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/devtron-labs/git-sensor/bean"
 	"github.com/devtron-labs/git-sensor/internals"
 	"github.com/devtron-labs/git-sensor/internals/sql"
 	"github.com/devtron-labs/git-sensor/internals/util"
@@ -39,7 +40,7 @@ type RepoManager interface {
 	AddRepo(gitCtx git.GitContext, material []*sql.GitMaterial) ([]*sql.GitMaterial, error)
 	UpdateRepo(gitCtx git.GitContext, material *sql.GitMaterial) (*sql.GitMaterial, error)
 	SavePipelineMaterial(gitCtx git.GitContext, material []*sql.CiPipelineMaterial) ([]*sql.CiPipelineMaterial, error)
-	ReloadAllRepo(gitCtx git.GitContext)
+	ReloadAllRepo(gitCtx git.GitContext, req *bean.ReloadAllMaterialQuery) (err error)
 	ResetRepo(gitCtx git.GitContext, materialId int) error
 	GetReleaseChanges(gitCtx git.GitContext, request *ReleaseChangesRequest) (*git.GitChanges, error)
 	GetCommitInfoForTag(gitCtx git.GitContext, request *git.CommitMetadataRequest) (*git.GitCommitBase, error)
@@ -202,7 +203,10 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
 		var repository *git.GitRepository
 		commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repository, pipelineMaterial.Value, "", "", fetchCount, material.CheckoutLocation, true)
 		//commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0)
-		if err == nil {
+		if gitCtx.Err() != nil {
+			impl.logger.Errorw("context error in getting commits", "err", gitCtx.Err())
+			return gitCtx.Err()
+		} else if err == nil {
 			impl.logger.Infow("commits found", "commit", commits)
 			b, err := json.Marshal(commits)
 			if err == nil {
@@ -361,7 +365,10 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
 	checkoutLocationForFetching := impl.repositoryManager.GetCheckoutLocation(gitCtx, material, gitProvider.Url, checkoutPath)
 
 	err = impl.repositoryManager.Add(gitCtx, material.GitProviderId, checkoutPath, material.Url, gitProvider.AuthMode, gitProvider.SshPrivateKey)
-	if err == nil {
+	if gitCtx.Err() != nil {
+		impl.logger.Errorw("context error in git checkout", "err", gitCtx.Err())
+		return material, gitCtx.Err()
+	} else if err == nil {
 		material.CheckoutLocation = checkoutLocationForFetching
 		material.CheckoutStatus = true
 	} else {
@@ -386,18 +393,38 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
 	return material, nil
 }
 
-func (impl RepoManagerImpl) ReloadAllRepo(gitCtx git.GitContext) {
-	materials, err := impl.materialRepository.FindAll()
-	if err != nil {
-		impl.logger.Errorw("error in reloading materials")
+func (impl RepoManagerImpl) ReloadAllRepo(gitCtx git.GitContext, req *bean.ReloadAllMaterialQuery) (err error) {
+	var materials []*sql.GitMaterial
+	if req != nil {
+		materials, err = impl.materialRepository.FindInRage(req.Start, req.End)
+		if err != nil {
+			impl.logger.Errorw(bean.GetReloadAllLog("error in getting reload materials"), "err", err)
+			return err
+		}
+	} else {
+		materials, err = impl.materialRepository.FindAll()
+		if err != nil {
+			impl.logger.Errorw(bean.GetReloadAllLog("error in getting reload materials"), "err", err)
+			return err
+		}
 	}
+
 	for _, material := range materials {
-		if _, err := impl.checkoutRepo(gitCtx, material); err != nil {
-			impl.logger.Errorw("error in checkout", "material", material, "err", err)
+		impl.logger.Infow(bean.GetReloadAllLog("performing material checkout for"), "materialId", material.Id)
+		_, err = impl.checkoutRepo(gitCtx, material)
+		if gitCtx.Err() != nil {
+			impl.logger.Errorw(bean.GetReloadAllLog("error in material checkout"), "materialId", material.Id, "err", gitCtx.Err())
+			return gitCtx.Err()
+		} else if err != nil {
+			impl.logger.Errorw(bean.GetReloadAllLog("context error in material while checkout"), "materialId", material.Id, "err", err)
+			// skipping for other materials to be processed
+		} else {
+			impl.logger.Infow(bean.GetReloadAllLog("successfully checked out for material"), "materialId", material.Id)
 		}
-
 	}
+	return nil
 }
+
 func (impl RepoManagerImpl) ResetRepo(gitCtx git.GitContext, materialId int) error {
 	material, err := impl.materialRepository.FindById(materialId)
 	if err != nil {