Merge branch 'main' into flux-app-list

README.md

@@ -187,14 +187,8 @@ Devtron is built on some of the most trusted and loved technologies:
 
 ## :muscle: Trusted By
 
-Devtron is trusted by Enterprises and Communities all across the globe:
-<br>
+Devtron is trusted by communities all across the globe. The list of organizations using Devtron can be found [here](./USERS.MD).
 
-- [Delhivery:](https://www.delhivery.com/) Delhivery is an Indian delivery and e-commerce logistics company, that provides end-to-end Supply Chain solutions through cutting-edge technology
-- [BharatPe:](https://bharatpe.com/) Bharatpe is an Indian fintech company that offers a range of products including interoperable QR codes for UPI payments, POS machines for card acceptance, and small business financing
-- [Livspace:](https://www.livspace.com/in) Livspace is a home interior and renovation company, that provides interior design and renovation services in Singapore and India
-- [Moglix:](https://www.moglix.com/) Moglix is an industrial B2B marketplace and an e-commerce platform for industrial tools and equipment, used largely by businesses in India
-- [Xoxoday:](https://www.xoxoday.com/) Xoxoday provides technology infrastructure to enable businesses to automate rewards, incentives & payouts for employees, customers & channel partners
 
 ## :question: FAQs & Troubleshooting
 

USERS.md

@@ -0,0 +1,26 @@
+# Who is using Devtron?
+
+As the community grows, we want to keep track of the users and organizations using Devtron. If you're using Devtron, please raise a PR to add your organization name and a link to your webpage.
+
+Currently, Devtron is being used by the following organizations:
+
+1. [73strings](https://www.73strings.com/)
+2. [Ather Energy](https://www.atherenergy.com/)
+3. [BharatPe](https://bharatpe.com/)
+4. [Birdeye](https://birdeye.com/)
+5. [Bluecopa](https://www.bluecopa.com/) 
+6. [Chitale Bandhu](https://www.chitalebandhu.in/) 
+7. [Centricity](https://centricity.co.in/) 
+8. [Cyble](https://cyble.com/) 
+9. [Delhivery](https://www.delhivery.com/)
+10. [KHEL Group](https://thekhelgroup.com/)
+11. [Lemnisk](https://www.lemnisk.co/) 
+12. [OTPLess](https://otpless.com/)
+14. [Spinny](https://www.spinny.com/)
+15. [Tata 1Mg](https://www.1mg.com/)
+16. [TravClan](https://www.travclan.com/) 
+17. [Unity Small Finance Bank](https://theunitybank.com/)
+18. [Xoxoday](https://www.xoxoday.com/)
+
+
+

api/appStore/InstalledAppRestHandler.go

@@ -28,6 +28,7 @@ import (
 	util3 "github.com/devtron-labs/devtron/pkg/appStore/util"
 	"github.com/devtron-labs/devtron/pkg/bean"
 	"net/http"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -363,6 +364,8 @@ func (handler *InstalledAppRestHandlerImpl) DeployBulk(w http.ResponseWriter, r
 		common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), nil, http.StatusForbidden)
 		return
 	}
+	charts, authRes := handler.checkForHelmDeployAuth(request, token)
+	request.ChartGroupInstallChartRequest = charts
 	//RBAC block ends here
 
 	visited := make(map[string]bool)
@@ -391,10 +394,116 @@ func (handler *InstalledAppRestHandlerImpl) DeployBulk(w http.ResponseWriter, r
 		handler.Logger.Errorw("service err, DeployBulk", "err", err, "payload", request)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
 		return
+	} else {
+		res = authRes
 	}
 	common.WriteJsonResp(w, err, res, http.StatusOK)
 }
 
+func (handler *InstalledAppRestHandlerImpl) checkForHelmDeployAuth(request chartGroup.ChartGroupInstallRequest, token string) ([]*chartGroup.ChartGroupInstallChartRequest, *chartGroup.ChartGroupInstallAppRes) {
+	//the value of this map is array of integer because the GetHelmObjectByProjectIdAndEnvId method may return "//" for error cases
+	//so different environments may contain same object, to handle that we are using (map[string] []int)
+	rbacObjectToEnvIdMap1 := make(map[string][]int)
+	rbacObjectToEnvIdMap2 := make(map[string][]int)
+
+	rbacObjectArray1 := make([]string, 0)
+	rbacObjectArray2 := make([]string, 0)
+
+	envIdToChartGroupInstallChartRequest := make(map[int][]*chartGroup.ChartGroupInstallChartRequest)
+
+	for _, chartGroupInstall := range request.ChartGroupInstallChartRequest {
+		envIdToChartGroupInstallChartRequest[chartGroupInstall.EnvironmentId] = append(envIdToChartGroupInstallChartRequest[chartGroupInstall.EnvironmentId], chartGroupInstall)
+		rbacObject1, rbacObject2 := handler.enforcerUtil.GetHelmObjectByProjectIdAndEnvId(request.ProjectId, chartGroupInstall.EnvironmentId)
+		_, ok := rbacObjectToEnvIdMap1[rbacObject1]
+		if !ok {
+			rbacObjectToEnvIdMap1[rbacObject1] = make([]int, 0)
+		}
+		rbacObjectToEnvIdMap1[rbacObject1] = append(rbacObjectToEnvIdMap1[rbacObject1], chartGroupInstall.EnvironmentId)
+		rbacObjectArray1 = append(rbacObjectArray1, rbacObject1)
+		_, ok = rbacObjectToEnvIdMap2[rbacObject2]
+		if !ok {
+			rbacObjectToEnvIdMap2[rbacObject2] = make([]int, 0)
+		}
+		rbacObjectToEnvIdMap2[rbacObject2] = append(rbacObjectToEnvIdMap2[rbacObject2], chartGroupInstall.EnvironmentId)
+		rbacObjectArray2 = append(rbacObjectArray2, rbacObject2)
+	}
+	resultObjectMap1 := handler.enforcer.EnforceInBatch(token, casbin.ResourceHelmApp, casbin.ActionCreate, rbacObjectArray1)
+	resultObjectMap2 := handler.enforcer.EnforceInBatch(token, casbin.ResourceHelmApp, casbin.ActionCreate, rbacObjectArray2)
+
+	authorizedEnvIdSet := make(map[int]bool)
+
+	//O(n) time loop , at max we will only iterate through all the envs
+	for obj, ok := range resultObjectMap1 {
+		if ok {
+			envIds := rbacObjectToEnvIdMap1[obj]
+			for _, envId := range envIds {
+				authorizedEnvIdSet[envId] = true
+			}
+		}
+	}
+	for obj, ok := range resultObjectMap2 {
+		if ok {
+			envIds := rbacObjectToEnvIdMap2[obj]
+			for _, envId := range envIds {
+				authorizedEnvIdSet[envId] = true
+			}
+		}
+	}
+	authorizedChartGroupInstallRequests := make([]*chartGroup.ChartGroupInstallChartRequest, 0)
+	for envId, _ := range authorizedEnvIdSet {
+		authorizedChartGroupInstall := envIdToChartGroupInstallChartRequest[envId]
+		for _, authChartGroup := range authorizedChartGroupInstall {
+			authorizedChartGroupInstallRequests = append(authorizedChartGroupInstallRequests, authChartGroup)
+		}
+	}
+	unauthorizedChartGroupInstallRequests := make([]*chartGroup.ChartGroupInstallChartRequest, 0)
+
+	for _, req := range request.ChartGroupInstallChartRequest {
+		isAuthorized := false
+		for _, authReq := range authorizedChartGroupInstallRequests {
+			if reflect.DeepEqual(req, authReq) {
+				isAuthorized = true
+				break
+			}
+		}
+		if !isAuthorized {
+			unauthorizedChartGroupInstallRequests = append(unauthorizedChartGroupInstallRequests, req)
+		}
+	}
+
+	// Create slices for ChartGroupInstallMetadata
+	authorizedMetadata := make([]chartGroup.ChartGroupInstallMetadata, 0)
+	unauthorizedMetadata := make([]chartGroup.ChartGroupInstallMetadata, 0)
+
+	for _, req := range authorizedChartGroupInstallRequests {
+		metadata := handler.getChartGroupInstallMetadata(req, string(chartGroup.StatusSuccess), string(chartGroup.ReasonTriggered))
+		authorizedMetadata = append(authorizedMetadata, metadata)
+	}
+
+	for _, req := range unauthorizedChartGroupInstallRequests {
+		metadata := handler.getChartGroupInstallMetadata(req, string(chartGroup.StatusFailed), string(chartGroup.ReasonNotAuthorize))
+		unauthorizedMetadata = append(unauthorizedMetadata, metadata)
+	}
+	unauthorizeCount := len(unauthorizedChartGroupInstallRequests)
+	totalCount := len(request.ChartGroupInstallChartRequest)
+	// Combine all metadata into a single ChartGroupInstallAppRes
+	chartGroupInstallAppRes := &chartGroup.ChartGroupInstallAppRes{
+		ChartGroupInstallMetadata: append(authorizedMetadata, unauthorizedMetadata...),
+		Summary:                   fmt.Sprintf(chartGroup.FAILED_TO_TRIGGER, unauthorizeCount, totalCount),
+	}
+	return authorizedChartGroupInstallRequests, chartGroupInstallAppRes
+}
+
+func (handler *InstalledAppRestHandlerImpl) getChartGroupInstallMetadata(req *chartGroup.ChartGroupInstallChartRequest, triggerStatus string, reason string) chartGroup.ChartGroupInstallMetadata {
+	metadata := chartGroup.ChartGroupInstallMetadata{
+		AppName:       req.AppName,
+		EnvironmentId: req.EnvironmentId,
+		TriggerStatus: triggerStatus,
+		Reason:        reason,
+	}
+	return metadata
+}
+
 func (handler *InstalledAppRestHandlerImpl) CheckAppExists(w http.ResponseWriter, r *http.Request) {
 	userId, err := handler.userAuthService.GetLoggedInUser(r)
 	if userId == 0 || err != nil {

pkg/appStore/chartGroup/bean.go

@@ -33,6 +33,23 @@ type ChartGroupInstallChartRequest struct {
 	ReferenceValueKind string `json:"referenceValueKind, omitempty" validate:"oneof=DEFAULT TEMPLATE DEPLOYED"`
 	ChartGroupEntryId  int    `json:"chartGroupEntryId"` //optional
 }
-
+type ChartGroupInstallMetadata struct {
+	AppName       string `json:"appName"`
+	EnvironmentId int    `json:"environmentId"`
+	TriggerStatus string `json:"triggerStatus"`
+	Reason        string `json:"reason"`
+}
 type ChartGroupInstallAppRes struct {
+	ChartGroupInstallMetadata []ChartGroupInstallMetadata `json:"chartGroupInstallMetadata"`
+	Summary                   string                      `json:"summary"`
 }
+type TriggerStatus string
+type Reason string
+
+const FAILED_TO_TRIGGER = "%d/%d failed to trigger"
+const (
+	StatusFailed       TriggerStatus = "failed"
+	StatusSuccess      TriggerStatus = "success"
+	ReasonNotAuthorize Reason        = "not authorized"
+	ReasonTriggered    Reason        = "triggered"
+)

pkg/auth/user/UserCommonService.go

@@ -572,13 +572,16 @@ func (impl UserCommonServiceImpl) CheckRbacForClusterEntity(cluster, namespace,
 		resourceObj = "*"
 	}
 
-	rbacResource := fmt.Sprintf("%s/%s/%s", strings.ToLower(cluster), strings.ToLower(namespaceObj), casbin.ResourceUser)
 	resourcesArray := strings.Split(resourceObj, ",")
-	for _, resourceVal := range resourcesArray {
-		rbacObject := fmt.Sprintf("%s/%s/%s", groupObj, kindObj, resourceVal)
-		allowed := managerAuth(rbacResource, token, rbacObject)
-		if !allowed {
-			return false
+	namespacesArray := strings.Split(namespaceObj, ",")
+	for _, namespaceInArray := range namespacesArray {
+		rbacResource := fmt.Sprintf("%s/%s/%s", strings.ToLower(cluster), strings.ToLower(namespaceInArray), casbin.ResourceUser)
+		for _, resourceVal := range resourcesArray {
+			rbacObject := fmt.Sprintf("%s/%s/%s", groupObj, kindObj, resourceVal)
+			allowed := managerAuth(rbacResource, token, rbacObject)
+			if !allowed {
+				return false
+			}
 		}
 	}
 	return true
@@ -690,8 +693,8 @@ func (impl UserCommonServiceImpl) GetUniqueKeyForAllEntity(role repository.RoleM
 		key = fmt.Sprintf("%s_%s_%s_%s", role.Team, role.Action, role.AccessType, role.Entity)
 	} else if len(role.Entity) > 0 {
 		if role.Entity == bean.CLUSTER_ENTITIY {
-			key = fmt.Sprintf("%s_%s_%s_%s_%s_%s", role.Entity, role.Action, role.Cluster,
-				role.Namespace, role.Group, role.Kind)
+			key = fmt.Sprintf("%s_%s_%s_%s_%s", role.Entity, role.Action, role.Cluster,
+				role.Group, role.Kind)
 		} else {
 			key = fmt.Sprintf("%s_%s", role.Entity, role.Action)
 		}

pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go

@@ -18,6 +18,7 @@ package chartRef
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -29,8 +30,10 @@ import (
 	util2 "github.com/devtron-labs/devtron/util"
 	dirCopy "github.com/otiai10/copy"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 	"io/ioutil"
 	"k8s.io/helm/pkg/chartutil"
+	chart2 "k8s.io/helm/pkg/proto/hapi/chart"
 	"os"
 	"path"
 	"path/filepath"
@@ -70,19 +73,21 @@ type ChartRefFileOpService interface {
 	GetRefChart(chartRefId int) (string, string, string, string, error)
 	ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*bean.ChartDataInfo, error)
 	GetChartInBytes(chartRefId int, deleteChart bool) ([]byte, error)
-	GetChartBytesInBulk(chartRefIds []int, deleteChart bool) (map[int][]byte, error)
+	GetChartBytesForApps(ctx context.Context, appIdToAppName map[int]string) (map[int][]byte, error)
 }
 
 type ChartRefServiceImpl struct {
 	logger               *zap.SugaredLogger
 	chartRefRepository   chartRepoRepository.ChartRefRepository
 	chartTemplateService util.ChartTemplateService
 	mergeUtil            util.MergeUtil
+	chartRepository      chartRepoRepository.ChartRepository
 }
 
 func NewChartRefServiceImpl(logger *zap.SugaredLogger,
 	chartRefRepository chartRepoRepository.ChartRefRepository,
 	chartTemplateService util.ChartTemplateService,
+	chartRepository chartRepoRepository.ChartRepository,
 	mergeUtil util.MergeUtil) *ChartRefServiceImpl {
 	// cache devtron reference charts list
 	devtronChartList, _ := chartRefRepository.FetchAllNonUserUploadedChartInfo()
@@ -92,6 +97,7 @@ func NewChartRefServiceImpl(logger *zap.SugaredLogger,
 		chartRefRepository:   chartRefRepository,
 		chartTemplateService: chartTemplateService,
 		mergeUtil:            mergeUtil,
+		chartRepository:      chartRepository,
 	}
 }
 
@@ -309,23 +315,83 @@ func (impl *ChartRefServiceImpl) extractChartInBytes(chartRef *chartRepoReposito
 	}
 	return manifestByteArr, nil
 }
-func (impl *ChartRefServiceImpl) GetChartBytesInBulk(chartRefIds []int, performCleanup bool) (map[int][]byte, error) {
+
+func (impl *ChartRefServiceImpl) getChartPath(chartRef *chartRepoRepository.ChartRef) (string, error) {
+	refChartPath := filepath.Join(bean.RefChartDirPath, chartRef.Location)
+	// For user uploaded charts ChartData will be retrieved from DB
+	if chartRef.ChartData != nil {
+		chartInfo, err := impl.ExtractChartIfMissing(chartRef.ChartData, bean.RefChartDirPath, chartRef.Location)
+		if chartInfo != nil && chartInfo.TemporaryFolder != "" {
+			err1 := os.RemoveAll(chartInfo.TemporaryFolder)
+			if err1 != nil {
+				impl.logger.Errorw("error in deleting temp dir ", "err", err)
+			}
+		}
+	} else {
+		// For Devtron reference charts the chart will be load from the directory location
+	}
+	return refChartPath, nil
+}
+
+func (impl *ChartRefServiceImpl) GetChartBytesForApps(ctx context.Context, appIdToAppName map[int]string) (map[int][]byte, error) {
+
+	appIds := maps.Keys(appIdToAppName)
+	charts, err := impl.chartRepository.FindLatestChartByAppIds(appIds)
+	if err != nil {
+		impl.logger.Errorw("error in fetching chart", "err", err, "appIds", appIds)
+		return nil, err
+	}
+
+	chartRefIdTOAppIds := make(map[int][]int)
+	var chartRefIds []int
+	chartRefToChartVersion := make(map[int]string)
+
+	for _, chart := range charts {
+		chartRefIds = append(chartRefIds, chart.ChartRefId)
+		chartRefToChartVersion[chart.ChartRefId] = chart.ChartVersion
+		refAppIds, ok := chartRefIdTOAppIds[chart.ChartRefId]
+		if !ok {
+			refAppIds = make([]int, 0)
+		}
+		refAppIds = append(refAppIds, chart.AppId)
+		chartRefIdTOAppIds[chart.ChartRefId] = refAppIds
+	}
+
 	chartRefs, err := impl.chartRefRepository.FindByIds(chartRefIds)
 	if err != nil {
 		impl.logger.Errorw("error getting chart data", "chartRefIds", chartRefIds, "err", err)
 		return nil, err
 	}
-	chartRefIdToBytes := make(map[int][]byte)
+
+	appIdToBytes := make(map[int][]byte)
+
+	// this loops run with O(len(apps)) T.C
 	for _, chartRef := range chartRefs {
-		chartInBytes, err := impl.extractChartInBytes(chartRef, performCleanup)
+		refChartPath, err := impl.getChartPath(chartRef)
 		if err != nil {
 			impl.logger.Errorw("error in converting chart to bytes", "chartRefId", chartRef.Id, "err", err)
 			return nil, err
 		}
-		chartRefIdToBytes[chartRef.Id] = chartInBytes
+
+		refAppIds := chartRefIdTOAppIds[chartRef.Id]
+		for _, appId := range refAppIds {
+			chartMetaData := &chart2.Metadata{
+				Name:    appIdToAppName[appId],
+				Version: chartRefToChartVersion[chartRef.Id],
+			}
+			tempReferenceTemplateDir, err := impl.chartTemplateService.BuildChart(ctx, chartMetaData, refChartPath)
+			if err != nil {
+				impl.logger.Errorw("error in building chart", "chartMetaData", chartMetaData, "refChartPath", refChartPath)
+				return nil, err
+			}
+			chartInBytes, err := impl.chartTemplateService.LoadChartInBytes(tempReferenceTemplateDir, true)
+			appIdToBytes[appId] = chartInBytes
+		}
+
 	}
-	return chartRefIdToBytes, nil
+	return appIdToBytes, nil
 }
+
 func (impl *ChartRefServiceImpl) FetchCustomChartsInfo() ([]*bean.ChartDto, error) {
 	resultsMetadata, err := impl.chartRefRepository.GetAllChartMetadata()
 	if err != nil {

pkg/generateManifest/DeploymentTemplateService.go

@@ -46,6 +46,7 @@ import (
 	"github.com/gammazero/workerpool"
 	"github.com/go-pg/pg"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 	"net/http"
 	"os"
 	"regexp"
@@ -422,6 +423,12 @@ func (impl DeploymentTemplateServiceImpl) GetRestartWorkloadData(ctx context.Con
 	var finalError error
 	for i, _ := range partitionedRequests {
 		req := partitionedRequests[i]
+		err = impl.setChartContent(ctx, req, appNameToId)
+		if err != nil {
+			impl.Logger.Errorw("error in setting chart content for apps", "appNames", maps.Keys(appNameToId), "err", err)
+			// continue processing next batch
+			continue
+		}
 		wp.Submit(func() {
 			resp, err := impl.helmAppClient.TemplateChartBulk(ctx, &gRPC.BulkInstallReleaseRequest{BulkInstallReleaseRequest: req})
 			if err != nil {