Main merge

Author: Ashish-devtron

Dockerfile

@@ -14,9 +14,10 @@ RUN apk add --no-cache ca-certificates
 RUN apk add git --no-cache
 RUN apk add openssh --no-cache
 COPY --from=build-env  /go/src/github.com/devtron-labs/git-sensor/git-sensor .
+COPY --from=build-env  /go/src/github.com/devtron-labs/git-sensor/scripts/ .
 
 RUN adduser -D devtron
 RUN chown -R devtron:devtron ./git-sensor
 USER devtron
 
-CMD ["./git-sensor"]
+CMD ["./git-sensor"]

Makefile

@@ -22,7 +22,8 @@ wire:
 	wire
 
 clean:
-	rm -f git-sensor
+	rm -rf git-sensor
+	export GOFLAGS=-buildvcs=false
 
 run: build
 	./git-sensor

api/GrpcHandler.go

@@ -198,7 +198,10 @@ func (impl *GrpcHandlerImpl) FetchChanges(ctx context.Context, req *pb.FetchScmC
 	if res.Commits != nil {
 		pbGitCommits = make([]*pb.GitCommit, 0, len(res.Commits))
 		for _, item := range res.Commits {
-
+			item.TruncateMessageIfExceedsMaxLength()
+			if !item.IsMessageValidUTF8() {
+				item.FixInvalidUTF8Message()
+			}
 			mappedCommit, err := impl.mapGitCommit(item)
 			if err != nil {
 				impl.logger.Debugw("failed to map git commit from bean to proto specified type",
@@ -515,8 +518,9 @@ func (impl *GrpcHandlerImpl) GetAllWebhookEventConfigForHost(ctx context.Context
 
 		return nil, err
 	}
+	webhookConfig := &pb.WebhookEventConfigResponse{}
 	if res == nil {
-		return nil, nil
+		return webhookConfig, nil
 	}
 
 	// Mapping response

internal/Configuration.go

@@ -3,7 +3,10 @@ package internal
 import "github.com/caarlos0/env"
 
 type Configuration struct {
-	CommitStatsTimeoutInSec int `env:"COMMIT_STATS_TIMEOUT_IN_SEC" envDefault:"2"`
+	CommitStatsTimeoutInSec int  `env:"COMMIT_STATS_TIMEOUT_IN_SEC" envDefault:"2"`
+	EnableFileStats         bool `env:"ENABLE_FILE_STATS" envDefault:"false"`
+	GitHistoryCount         int  `env:"GIT_HISTORY_COUNT" envDefault:"15"`
+	MinLimit                int  `env:"MIN_LIMIT_FOR_PVC" envDefault:"1"` // in MB
 }
 
 func ParseConfiguration() (*Configuration, error) {

pkg/RepoManages.go

@@ -65,6 +65,7 @@ type RepoManagerImpl struct {
 	webhookEventDataMappingRepository             sql.WebhookEventDataMappingRepository
 	webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository
 	webhookEventBeanConverter                     git.WebhookEventBeanConverter
+	configuration                                 *internal.Configuration
 }
 
 func NewRepoManagerImpl(
@@ -79,6 +80,7 @@ func NewRepoManagerImpl(
 	webhookEventDataMappingRepository sql.WebhookEventDataMappingRepository,
 	webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository,
 	webhookEventBeanConverter git.WebhookEventBeanConverter,
+	configuration *internal.Configuration,
 ) *RepoManagerImpl {
 	return &RepoManagerImpl{
 		logger:                            logger,
@@ -93,6 +95,7 @@ func NewRepoManagerImpl(
 		webhookEventDataMappingRepository: webhookEventDataMappingRepository,
 		webhookEventDataMappingFilterResultRepository: webhookEventDataMappingFilterResultRepository,
 		webhookEventBeanConverter:                     webhookEventBeanConverter,
+		configuration:                                 configuration,
 	}
 }
 
@@ -187,7 +190,7 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(materials []*sql.CiPipe
 			continue
 		}
 
-		commits, err := impl.repositoryManager.ChangesSince(material.CheckoutLocation, pipelineMaterial.Value, "", "", 0)
+		commits, err := impl.repositoryManager.ChangesSince(material.CheckoutLocation, pipelineMaterial.Value, "", "", impl.configuration.GitHistoryCount)
 		//commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0)
 		if err == nil {
 			impl.logger.Infow("commits found", "commit", commits)
@@ -341,7 +344,11 @@ func (impl RepoManagerImpl) checkoutMaterial(material *sql.GitMaterial) (*sql.Gi
 	if err != nil {
 		return material, err
 	}
-	err = impl.repositoryManager.Add(material.GitProviderId, checkoutPath, material.Url, userName, password, gitProvider.AuthMode, gitProvider.SshPrivateKey)
+	gitContext := &git.GitContext{
+		Username: userName,
+		Password: password,
+	}
+	err = impl.repositoryManager.Add(material.GitProviderId, checkoutPath, material.Url, gitContext, gitProvider.AuthMode, gitProvider.SshPrivateKey)
 	if err == nil {
 		material.CheckoutLocation = checkoutPath
 		material.CheckoutStatus = true
@@ -617,17 +624,38 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(pipelineMaterialId int, bra
 	}()
 
 	userName, password, err := git.GetUserNamePassword(gitMaterial.GitProvider)
-	updated, repo, err := impl.repositoryManager.Fetch(userName, password, gitMaterial.Url, gitMaterial.CheckoutLocation)
+	gitContext := &git.GitContext{
+		Username: userName,
+		Password: password,
+	}
+	updated, repo, err := impl.repositoryManager.Fetch(gitContext, gitMaterial.Url, gitMaterial.CheckoutLocation, gitMaterial)
+	if !updated {
+		impl.logger.Warn("repository is up to date")
+	}
+	if err == nil {
+		gitMaterial.CheckoutStatus = true
+	} else {
+		gitMaterial.CheckoutStatus = false
+		gitMaterial.CheckoutMsgAny = err.Error()
+		gitMaterial.FetchErrorMessage = err.Error()
+
+		impl.logger.Errorw("error in fetching the repository ", "gitMaterial", gitMaterial, "err", err)
+		return nil, err
+	}
 
+	err = impl.materialRepository.Update(gitMaterial)
 	if err != nil {
-		impl.logger.Errorw("error in fetching the repository ", "err", err)
+		impl.logger.Errorw("error in updating material repo", "err", err, "material", gitMaterial)
 		return nil, err
 	}
-	if !updated {
-		impl.logger.Warn("repository is up to date")
+	ciPipelineMaterial, err := impl.ciPipelineMaterialRepository.FindByGitMaterialId(gitMaterial.Id)
+	if err != nil {
+		impl.logger.Errorw("unable to load material", "err", err, "ciPipelineMaterial", ciPipelineMaterial)
+		return nil, err
 	}
+	err = impl.updatePipelineMaterialCommit(ciPipelineMaterial)
 	if err != nil {
-		impl.logger.Errorw("error in fetching the repository ", "err", err)
+		impl.logger.Errorw("error in updating pipeline material", "err", err)
 		return nil, err
 	}
 

pkg/git/Bean.go

@@ -20,6 +20,7 @@ import (
 	"github.com/devtron-labs/git-sensor/internal/sql"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"time"
+	"unicode/utf8"
 )
 
 type FetchScmChangesRequest struct {
@@ -64,6 +65,29 @@ type GitCommit struct {
 	Excluded    bool              `json:",omitempty"`
 }
 
+func (gitCommit *GitCommit) TruncateMessageIfExceedsMaxLength() {
+	maxLength := 1024
+	if len(gitCommit.Message) > maxLength {
+		gitCommit.Message = gitCommit.Message[:maxLength-3] + "..."
+	}
+}
+
+// IsMessageValidUTF8 checks if a string is valid UTF-8.
+func (gitCommit *GitCommit) IsMessageValidUTF8() bool {
+	return utf8.ValidString(gitCommit.Message)
+}
+
+// FixInvalidUTF8Message replaces invalid UTF-8 sequences with the replacement character (U+FFFD).
+func (gitCommit *GitCommit) FixInvalidUTF8Message() {
+	invalidUTF8 := []rune(gitCommit.Message)
+	for i, r := range invalidUTF8 {
+		if !utf8.ValidRune(r) {
+			invalidUTF8[i] = utf8.RuneError
+		}
+	}
+	gitCommit.Message = string(invalidUTF8)
+}
+
 type WebhookAndCiData struct {
 	ExtraEnvironmentVariables map[string]string `json:"extraEnvironmentVariables"` // extra env variables which will be used for CI
 	WebhookData               *WebhookData      `json:"webhookData"`

pkg/git/GitCliUtil.go

@@ -1,6 +1,7 @@
 package git
 
 import (
+	"context"
 	"fmt"
 	"go.uber.org/zap"
 	"gopkg.in/src-d/go-git.v4"
@@ -10,6 +11,13 @@ import (
 	"strings"
 )
 
+type GitContext struct {
+	context.Context // Embedding original Go context
+	Username        string
+	Password        string
+	CloningMode     string
+}
+
 type GitUtil struct {
 	logger *zap.SugaredLogger
 }
@@ -22,10 +30,10 @@ func NewGitUtil(logger *zap.SugaredLogger) *GitUtil {
 
 const GIT_ASK_PASS = "/git-ask-pass.sh"
 
-func (impl *GitUtil) Fetch(rootDir string, username string, password string) (response, errMsg string, err error) {
+func (impl *GitUtil) Fetch(gitContext *GitContext, rootDir string) (response, errMsg string, err error) {
 	impl.logger.Debugw("git fetch ", "location", rootDir)
 	cmd := exec.Command("git", "-C", rootDir, "fetch", "origin", "--tags", "--force")
-	output, errMsg, err := impl.runCommandWithCred(cmd, username, password)
+	output, errMsg, err := impl.runCommandWithCred(cmd, gitContext.Username, gitContext.Password)
 	impl.logger.Debugw("fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	return output, errMsg, err
 }