tls support for git provider in git cli

Author: Shivam-nagar23

api/GrpcHandler.go

@@ -81,6 +81,9 @@ func (impl *GrpcHandlerImpl) SaveGitProvider(ctx context.Context, req *pb.GitPro
 		SshPrivateKey: req.SshPrivateKey,
 		AuthMode:      sql.AuthMode(req.AuthMode),
 		Active:        req.Active,
+		CaCert:        req.CaCert,
+		TlsCert:       req.TlsCert,
+		TlsKey:        req.TlsKey,
 	}
 
 	_, err := impl.repositoryManager.SaveGitProvider(gitProvider)

go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/caarlos0/env v3.5.0+incompatible
 	github.com/devtron-labs/common-lib v0.0.18-0.20240520062828-c6c38c3f135e
-	github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb
+	github.com/devtron-labs/protos v0.0.3-0.20240613123522-f9b02995eb55
 	github.com/gammazero/workerpool v0.0.0-20200206003619-019d125201ab
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/go-pg/pg v6.15.1+incompatible

go.sum

@@ -31,6 +31,8 @@ github.com/devtron-labs/common-lib v0.0.18-0.20240520062828-c6c38c3f135e h1:feRK
 github.com/devtron-labs/common-lib v0.0.18-0.20240520062828-c6c38c3f135e/go.mod h1:deAcJ5IjUjM6ozZQLJEgPWDUA0mKa632LBsKx8uM9TE=
 github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb h1:CkfQQgZc950/hTPqtQSiHV2RmZgkBLGCzwR02FZYjAU=
 github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb/go.mod h1:pjLjgoa1GzbkOkvbMyP4SAKsaiK7eG6GoQCNauG03JA=
+github.com/devtron-labs/protos v0.0.3-0.20240613123522-f9b02995eb55 h1:yB2S48rk6eNzva8tdxhWbpQ3jsJH+EhMKKE8bgGxdzk=
+github.com/devtron-labs/protos v0.0.3-0.20240613123522-f9b02995eb55/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=

internals/sql/GitProviderRepository.go

@@ -38,6 +38,9 @@ type GitProvider struct {
 	AccessToken   string   `sql:"access_token"`
 	AuthMode      AuthMode `sql:"auth_mode,notnull"`
 	Active        bool     `sql:"active,notnull"`
+	TlsCert       string   `sql:"tls_cert"`
+	TlsKey        string   `sql:"tls_key"`
+	CaCert        string   `sql:"ca_cert"`
 	//models.AuditLog
 }
 

pkg/RepoManages.go

@@ -196,6 +196,8 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
 		}
 
 		gitCtx = gitCtx.WithCredentials(material.GitProvider.UserName, material.GitProvider.Password).
+			WithTLSData(material.GitProvider.CaCert, material.GitProvider.TlsKey, material.GitProvider.TlsCert).
+			WithGitProviderId(material.GitProviderId).
 			WithCloningMode(impl.configuration.CloningMode)
 
 		fetchCount := impl.configuration.GitHistoryCount
@@ -352,6 +354,8 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
 	}
 
 	gitCtx = gitCtx.WithCredentials(userName, password).
+		WithTLSData(material.GitProvider.CaCert, material.GitProvider.TlsKey, material.GitProvider.TlsCert).
+		WithGitProviderId(material.GitProviderId).
 		WithCloningMode(impl.configuration.CloningMode)
 
 	checkoutPath, _, _, err := impl.repositoryManager.GetCheckoutLocationFromGitUrl(material, gitCtx.CloningMode)
@@ -639,6 +643,8 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe
 	userName, password, err := git.GetUserNamePassword(gitMaterial.GitProvider)
 
 	gitCtx = gitCtx.WithCredentials(userName, password).
+		WithTLSData(gitMaterial.GitProvider.CaCert, gitMaterial.GitProvider.TlsKey, gitMaterial.GitProvider.TlsCert).
+		WithGitProviderId(gitMaterial.GitProviderId).
 		WithCloningMode(impl.configuration.CloningMode)
 
 	updated, repo, err := impl.repositoryManager.Fetch(gitCtx, gitMaterial.Url, gitMaterial.CheckoutLocation)
@@ -708,6 +714,8 @@ func (impl RepoManagerImpl) GetCommitMetadataForPipelineMaterial(gitCtx git.GitC
 	}
 
 	gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password).
+		WithTLSData(gitMaterial.GitProvider.CaCert, gitMaterial.GitProvider.TlsKey, gitMaterial.GitProvider.TlsCert).
+		WithGitProviderId(gitMaterial.GitProviderId).
 		WithCloningMode(impl.configuration.CloningMode)
 	// validate checkout status of gitMaterial
 	if !gitMaterial.CheckoutStatus {
@@ -764,6 +772,8 @@ func (impl RepoManagerImpl) GetReleaseChanges(gitCtx git.GitContext, request *Re
 	}()
 
 	gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password).
+		WithTLSData(gitMaterial.GitProvider.CaCert, gitMaterial.GitProvider.TlsKey, gitMaterial.GitProvider.TlsCert).
+		WithGitProviderId(gitMaterial.GitProviderId).
 		WithCloningMode(impl.configuration.CloningMode)
 
 	gitChanges, err := impl.repositoryManagerAnalytics.ChangesSinceByRepositoryForAnalytics(gitCtx, gitMaterial.CheckoutLocation, request.OldCommit, request.NewCommit)

pkg/git/Bean.go

@@ -333,3 +333,9 @@ type IteratorRequest struct {
 	FromCommitHash string
 	ToCommitHash   string
 }
+
+type TlsPathInfo struct {
+	CaCertPath  string
+	TlsKeyPath  string
+	TlsCertPath string
+}

pkg/git/GitBaseManager.go

@@ -110,12 +110,18 @@ func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (respon
 	impl.logger.Debugw("git fetch ", "location", rootDir)
 	cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force")
 	defer cancel()
-	output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password)
+	tlsPathInfo, err := createFilesForTlsData(gitCtx)
+	if err != nil {
+		//making it non-blocking
+		impl.logger.Errorw("error encountered in createFilesForTlsData", "err", err)
+	}
+	defer impl.deleteTlsFiles(tlsPathInfo)
+	output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password, tlsPathInfo)
 	if strings.Contains(output, LOCK_REF_MESSAGE) {
 		impl.logger.Info("error in fetch, pruning local refs and retrying", "rootDir", rootDir)
 		// running git remote prune origin and retrying fetch. gitHub issue - https://github.com/devtron-labs/devtron/issues/4605
 		pruneCmd, pruneCmdCancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "remote", "prune", "origin")
-		pruneOutput, pruneMsg, pruneErr := impl.runCommandWithCred(pruneCmd, gitCtx.Username, gitCtx.Password)
+		pruneOutput, pruneMsg, pruneErr := impl.runCommandWithCred(pruneCmd, gitCtx.Username, gitCtx.Password, tlsPathInfo)
 		defer pruneCmdCancel()
 		if pruneErr != nil {
 			impl.logger.Errorw("error in pruning local refs that do not exist at remote")
@@ -125,7 +131,7 @@ func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (respon
 		retryFetchCmd, retryFetchCancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force")
 		defer retryFetchCancel()
 
-		output, errMsg, err = impl.runCommandWithCred(retryFetchCmd, gitCtx.Username, gitCtx.Password)
+		output, errMsg, err = impl.runCommandWithCred(retryFetchCmd, gitCtx.Username, gitCtx.Password, tlsPathInfo)
 	}
 	impl.logger.Debugw("fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	return output, errMsg, err
@@ -165,12 +171,33 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st
 	return commits, nil
 }
 
-func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) {
+func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, password string, tlsPathInfo *TlsPathInfo) (response, errMsg string, err error) {
 	cmd.Env = append(os.Environ(),
 		fmt.Sprintf("GIT_ASKPASS=%s", GIT_ASK_PASS),
 		fmt.Sprintf("GIT_USERNAME=%s", userName),
 		fmt.Sprintf("GIT_PASSWORD=%s", password),
 	)
+	if tlsPathInfo != nil {
+		if tlsPathInfo.TlsKeyPath != "" && tlsPathInfo.TlsCertPath != "" {
+			cmd.Env = append(cmd.Env,
+				fmt.Sprintf("GIT_SSL_KEY=%s", tlsPathInfo.TlsKeyPath),
+				fmt.Sprintf("GIT_SSL_CERT=%s", tlsPathInfo.TlsCertPath))
+		}
+		if tlsPathInfo.CaCertPath != "" {
+			cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_SSL_CAINFO=%s", tlsPathInfo.CaCertPath))
+		}
+	}
+	return impl.runCommand(cmd)
+}
+
+func (impl *GitManagerBaseImpl) runCommandWithTlsData(cmd *exec.Cmd, tlsPathInfo *TlsPathInfo) (response, errMsg string, err error) {
+	if tlsPathInfo != nil {
+		cmd.Env = append(os.Environ(),
+			fmt.Sprintf("GIT_SSL_CAINFO=%s", tlsPathInfo.CaCertPath),
+			fmt.Sprintf("GIT_SSL_KEY=%s", tlsPathInfo.TlsKeyPath),
+			fmt.Sprintf("GIT_SSL_CERT=%s", tlsPathInfo.TlsCertPath),
+		)
+	}
 	return impl.runCommand(cmd)
 }
 
@@ -296,8 +323,13 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommits(gitCtx GitContext, o
 	}
 	cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--numstat", oldHash, newHash)
 	defer cancel()
-
-	output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password)
+	tlsPathInfo, err := createFilesForTlsData(gitCtx)
+	if err != nil {
+		//making it non-blocking
+		impl.logger.Errorw("error encountered in createFilesForTlsData", "err", err)
+	}
+	defer impl.deleteTlsFiles(tlsPathInfo)
+	output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password, tlsPathInfo)
 	impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
 	if err != nil || len(errMsg) > 0 {
 		impl.logger.Errorw("error in fetching fileStat diff btw commits: ", "oldHash", oldHash, "newHash", newHash, "checkoutPath", rootDir, "errorMsg", errMsg, "err", err)
@@ -333,6 +365,63 @@ func (impl *GitManagerBaseImpl) getCommandTimeout(command string) int {
 func (impl *GitManagerBaseImpl) ExecuteCustomCommand(gitContext GitContext, name string, arg ...string) (response, errMsg string, err error) {
 	cmd, cancel := impl.createCmdWithContext(gitContext, name, arg...)
 	defer cancel()
-	output, errMsg, err := impl.runCommandWithCred(cmd, gitContext.Username, gitContext.Password)
+	tlsPathInfo, err := createFilesForTlsData(gitContext)
+	if err != nil {
+		//making it non-blocking
+		impl.logger.Errorw("error encountered in createFilesForTlsData", "err", err)
+	}
+	defer impl.deleteTlsFiles(tlsPathInfo)
+	output, errMsg, err := impl.runCommandWithCred(cmd, gitContext.Username, gitContext.Password, tlsPathInfo)
 	return output, errMsg, err
 }
+
+func createFilesForTlsData(gitContext GitContext) (*TlsPathInfo, error) {
+	var tlsKeyFilePath string
+	var tlsCertFilePath string
+	var caCertFilePath string
+	var err error
+	if gitContext.TLSKey != "" && gitContext.TLSCertificate != "" {
+		tlsKeyFilePath, err = CreateTlsPathFilesWithData(gitContext.GitProviderId, gitContext.TLSKey, TLS_KEY_FILE_NAME)
+		if err != nil {
+			return nil, err
+		}
+		tlsCertFilePath, err = CreateTlsPathFilesWithData(gitContext.GitProviderId, gitContext.TLSCertificate, TLS_CERT_FILE_NAME)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if gitContext.CACert != "" {
+		caCertFilePath, err = CreateTlsPathFilesWithData(gitContext.GitProviderId, gitContext.CACert, CA_CERT_FILE_NAME)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return BuildTlsInfoPath(caCertFilePath, tlsKeyFilePath, tlsCertFilePath), nil
+
+}
+
+func (impl *GitManagerBaseImpl) deleteTlsFiles(pathInfo *TlsPathInfo) {
+	if pathInfo == nil {
+		return
+	}
+	if pathInfo.TlsKeyPath != "" {
+		err := DeleteAFileIfExists(pathInfo.TlsKeyPath)
+		if err != nil {
+			impl.logger.Errorw("error in deleting file", "tlsKeyPath", pathInfo.TlsKeyPath, "err", err)
+		}
+	}
+
+	if pathInfo.TlsCertPath != "" {
+		err := DeleteAFileIfExists(pathInfo.TlsCertPath)
+		if err != nil {
+			impl.logger.Errorw("error in deleting file", "TlsCertPath", pathInfo.TlsCertPath, "err", err)
+		}
+	}
+	if pathInfo.CaCertPath != "" {
+		err := DeleteAFileIfExists(pathInfo.CaCertPath)
+		if err != nil {
+			impl.logger.Errorw("error in deleting file", "CaCertPath", pathInfo.CaCertPath, "err", err)
+		}
+	}
+	return
+}

pkg/git/GitContextUtils.go

@@ -26,6 +26,10 @@ type GitContext struct {
 	Username        string
 	Password        string
 	CloningMode     string
+	CACert          string
+	TLSKey          string
+	TLSCertificate  string
+	GitProviderId   int
 }
 
 func (gitCtx GitContext) WithCredentials(Username string, Password string) GitContext {
@@ -34,6 +38,18 @@ func (gitCtx GitContext) WithCredentials(Username string, Password string) GitCo
 	return gitCtx
 }
 
+func (gitCtx GitContext) WithTLSData(caData string, tlsKey string, tlsCertificate string) GitContext {
+	gitCtx.CACert = caData
+	gitCtx.TLSKey = tlsKey
+	gitCtx.TLSCertificate = tlsCertificate
+	return gitCtx
+}
+
+func (gitCtx GitContext) WithGitProviderId(gitProviderId int) GitContext {
+	gitCtx.GitProviderId = gitProviderId
+	return gitCtx
+}
+
 func BuildGitContext(ctx context.Context) GitContext {
 	return GitContext{
 		Context: ctx,

pkg/git/Util.go

@@ -30,7 +30,11 @@ import (
 const (
 	GIT_BASE_DIR              = "/git-base/"
 	SSH_PRIVATE_KEY_DIR       = GIT_BASE_DIR + "ssh-keys/"
+	TLS_FILES_DIR             = GIT_BASE_DIR + "tls-files/"
 	SSH_PRIVATE_KEY_FILE_NAME = "ssh_pvt_key"
+	TLS_KEY_FILE_NAME         = "tls_key"
+	TLS_CERT_FILE_NAME        = "tls_cert"
+	CA_CERT_FILE_NAME         = "ca_cert"
 	CLONE_TIMEOUT_SEC         = 600
 	FETCH_TIMEOUT_SEC         = 30
 	GITHUB_PROVIDER           = "github.com"
@@ -122,6 +126,36 @@ func CreateOrUpdateSshPrivateKeyOnDisk(gitProviderId int, sshPrivateKeyContent s
 	return nil
 }
 
+func CreateTlsPathFilesWithData(gitProviderId int, content string, fileName string) (string, error) {
+	tlsFolderPath := path.Join(TLS_FILES_DIR, strconv.Itoa(gitProviderId))
+	tlsFilePath := path.Join(tlsFolderPath, fileName)
+
+	// if file exists then delete file
+	if _, err := os.Stat(tlsFilePath); os.IsExist(err) {
+		os.Remove(tlsFilePath)
+	}
+	// create dirs
+	err := os.MkdirAll(tlsFolderPath, 0755)
+	if err != nil {
+		return "", err
+	}
+
+	// create file with content
+	err = ioutil.WriteFile(tlsFilePath, []byte(content), 0600)
+	if err != nil {
+		return "", err
+	}
+	return tlsFilePath, nil
+}
+
+func DeleteAFileIfExists(path string) error {
+	if _, err := os.Stat(path); os.IsExist(err) {
+		err = os.Remove(path)
+		return err
+	}
+	return nil
+}
+
 // sample commitDiff :=4\t3\tModels/models.go\n2\t2\tRepository/Repository.go\n0\t2\main.go
 func getFileStat(commitDiff string) (FileStats, error) {
 	filestat := FileStats{}

pkg/git/Watcher.go

@@ -194,6 +194,8 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e
 	}
 	gitCtx := BuildGitContext(context.Background()).
 		WithCredentials(userName, password).
+		WithTLSData(gitProvider.CaCert, gitProvider.TlsKey, gitProvider.TlsCert).
+		WithGitProviderId(gitProvider.Id).
 		WithCloningMode(impl.configuration.CloningMode)
 
 	updated, repo, err := impl.FetchAndUpdateMaterial(gitCtx, material, location)