feat: add plugin via api (#3937)

* create and update added

* preparing plugin dtos

* create plugin support added

* update plugin support

* deep update plugin support for steps, stepvariable and stepvariableconditions

* delete plugin operation added

* fix for delete pipeline

* refactoring for updating pluginStep, pluginStepVariables and pluginStepVariableConditions

* pg no rows handling

* todo

* minor bug fixes

* one level code review incorporation

* 2nd level code review incorporation, handling errNoRows, extracting out json from sql obj, checking if plugin is used at some pipeline step, and minor refactoring

* create and update new plugin tags and plugin tag relations in bul operation incorporated

* bulk update of plugin variables and plugin step variable conditions incorporated

* minor fix for saving tags in bulk

* create and delete ScriptPathArgPortMapping support given in for adding plugin via API

* get ScriptPathArgPortMapping support given in for adding plugin via API

* final fixes

* several bug fixes after ist iter qa

* some code refactoring

* removing tag plugin mapping duplicacy

* removing plugin id validation in create req

* incorporated validation on plugin create req and update req

* changing enum value of ci cd and cicd

* code review comments incorporation (code refactoring)

* comment fix

* moved audit log to UtilStructs

api/restHandler/GlobalPluginRestHandler.go

@@ -1,11 +1,12 @@
 package restHandler
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/devtron-labs/devtron/api/restHandler/common"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
 	"github.com/devtron-labs/devtron/pkg/plugin"
-	"github.com/devtron-labs/devtron/pkg/plugin/repository"
+	"github.com/devtron-labs/devtron/pkg/user"
 	"github.com/devtron-labs/devtron/pkg/user/casbin"
 	"github.com/devtron-labs/devtron/util/rbac"
 	"github.com/gorilla/mux"
@@ -15,19 +16,25 @@ import (
 )
 
 type GlobalPluginRestHandler interface {
+	PatchPlugin(w http.ResponseWriter, r *http.Request)
+
 	GetAllGlobalVariables(w http.ResponseWriter, r *http.Request)
 	ListAllPlugins(w http.ResponseWriter, r *http.Request)
 	GetPluginDetailById(w http.ResponseWriter, r *http.Request)
+	GetDetailedPluginInfoByPluginId(w http.ResponseWriter, r *http.Request)
+	GetAllDetailedPluginInfo(w http.ResponseWriter, r *http.Request)
 }
 
 func NewGlobalPluginRestHandler(logger *zap.SugaredLogger, globalPluginService plugin.GlobalPluginService,
-	enforcerUtil rbac.EnforcerUtil, enforcer casbin.Enforcer, pipelineBuilder pipeline.PipelineBuilder) *GlobalPluginRestHandlerImpl {
+	enforcerUtil rbac.EnforcerUtil, enforcer casbin.Enforcer, pipelineBuilder pipeline.PipelineBuilder,
+	userService user.UserService) *GlobalPluginRestHandlerImpl {
 	return &GlobalPluginRestHandlerImpl{
 		logger:              logger,
 		globalPluginService: globalPluginService,
 		enforcerUtil:        enforcerUtil,
 		enforcer:            enforcer,
 		pipelineBuilder:     pipelineBuilder,
+		userService:         userService,
 	}
 }
 
@@ -37,6 +44,97 @@ type GlobalPluginRestHandlerImpl struct {
 	enforcerUtil        rbac.EnforcerUtil
 	enforcer            casbin.Enforcer
 	pipelineBuilder     pipeline.PipelineBuilder
+	userService         user.UserService
+}
+
+func (handler *GlobalPluginRestHandlerImpl) PatchPlugin(w http.ResponseWriter, r *http.Request) {
+	decoder := json.NewDecoder(r.Body)
+	userId, err := handler.userService.GetLoggedInUser(r)
+	if userId == 0 || err != nil {
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized)
+		return
+	}
+	var pluginDataDto plugin.PluginMetadataDto
+	err = decoder.Decode(&pluginDataDto)
+	if err != nil {
+		handler.logger.Errorw("request err, PatchPlugin", "error", err, "payload", pluginDataDto)
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	handler.logger.Infow("request payload received for patching plugins", pluginDataDto, "userId", userId)
+	// RBAC enforcer applying
+	isSuperAdmin, err := handler.userService.IsSuperAdmin(int(userId))
+	if !isSuperAdmin || err != nil {
+		if err != nil {
+			handler.logger.Errorw("request err, CheckSuperAdmin", "err", err, "isSuperAdmin", isSuperAdmin)
+		}
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden)
+		return
+	}
+	//RBAC enforcer Ends
+	pluginData, err := handler.globalPluginService.PatchPlugin(&pluginDataDto, userId)
+	if err != nil {
+		handler.logger.Errorw("error in patching plugin data", "action", pluginDataDto.Action, "pluginMetadataPayloadDto", pluginDataDto, "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	common.WriteJsonResp(w, nil, pluginData, http.StatusOK)
+
+}
+func (handler *GlobalPluginRestHandlerImpl) GetDetailedPluginInfoByPluginId(w http.ResponseWriter, r *http.Request) {
+	userId, err := handler.userService.GetLoggedInUser(r)
+	if userId == 0 || err != nil {
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized)
+		return
+	}
+	vars := mux.Vars(r)
+	pluginId, err := strconv.Atoi(vars["pluginId"])
+	if err != nil {
+		handler.logger.Errorw("error in converting from string to integer", "pluginId", vars["pluginId"], "userId", userId, "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	// RBAC enforcer applying
+	isSuperAdmin, err := handler.userService.IsSuperAdmin(int(userId))
+	if !isSuperAdmin || err != nil {
+		if err != nil {
+			handler.logger.Errorw("request err, CheckSuperAdmin", "err", err, "isSuperAdmin", isSuperAdmin)
+		}
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden)
+		return
+	}
+	//RBAC enforcer Ends
+	pluginMetaData, err := handler.globalPluginService.GetDetailedPluginInfoByPluginId(pluginId)
+	if err != nil {
+		handler.logger.Errorw("error in getting plugin metadata", "pluginId", pluginId, "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	common.WriteJsonResp(w, nil, pluginMetaData, http.StatusOK)
+}
+func (handler *GlobalPluginRestHandlerImpl) GetAllDetailedPluginInfo(w http.ResponseWriter, r *http.Request) {
+	userId, err := handler.userService.GetLoggedInUser(r)
+	if userId == 0 || err != nil {
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized)
+		return
+	}
+	// RBAC enforcer applying
+	isSuperAdmin, err := handler.userService.IsSuperAdmin(int(userId))
+	if !isSuperAdmin || err != nil {
+		if err != nil {
+			handler.logger.Errorw("request err, CheckSuperAdmin", "err", err, "isSuperAdmin", isSuperAdmin)
+		}
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden)
+		return
+	}
+	//RBAC enforcer Ends
+	pluginMetaData, err := handler.globalPluginService.GetAllDetailedPluginInfo()
+	if err != nil {
+		handler.logger.Errorw("error in getting all plugins metadata", "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	common.WriteJsonResp(w, nil, pluginMetaData, http.StatusOK)
 }
 
 func (handler *GlobalPluginRestHandlerImpl) GetAllGlobalVariables(w http.ResponseWriter, r *http.Request) {
@@ -94,20 +192,11 @@ func (handler *GlobalPluginRestHandlerImpl) ListAllPlugins(w http.ResponseWriter
 		return
 	}
 	var plugins []*plugin.PluginListComponentDto
-	if stageType == repository.CD_STAGE_TYPE {
-		plugins, err = handler.globalPluginService.ListAllPlugins(repository.CD)
-		if err != nil {
-			handler.logger.Errorw("error in getting cd plugin list", "err", err)
-			common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
-			return
-		}
-	} else {
-		plugins, err = handler.globalPluginService.ListAllPlugins(repository.CI)
-		if err != nil {
-			handler.logger.Errorw("error in getting ci plugin list", "err", err)
-			common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
-			return
-		}
+	plugins, err = handler.globalPluginService.ListAllPlugins(stageType)
+	if err != nil {
+		handler.logger.Errorw("error in getting cd plugin list", "err", err)
+		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
 	}
 
 	common.WriteJsonResp(w, err, plugins, http.StatusOK)

api/router/GlobalPluginRouter.go

@@ -23,12 +23,19 @@ type GlobalPluginRouterImpl struct {
 }
 
 func (impl *GlobalPluginRouterImpl) initGlobalPluginRouter(globalPluginRouter *mux.Router) {
-	globalPluginRouter.Path("/global/list/global-variable").
+	globalPluginRouter.Path("").
+		HandlerFunc(impl.globalPluginRestHandler.PatchPlugin).Methods("POST")
+	globalPluginRouter.Path("/detail/all").
+		HandlerFunc(impl.globalPluginRestHandler.GetAllDetailedPluginInfo).Methods("GET")
+	globalPluginRouter.Path("/detail/{pluginId}").
+		HandlerFunc(impl.globalPluginRestHandler.GetDetailedPluginInfoByPluginId).Methods("GET")
+
+	globalPluginRouter.Path("/list/global-variable").
 		HandlerFunc(impl.globalPluginRestHandler.GetAllGlobalVariables).Methods("GET")
 
-	globalPluginRouter.Path("/global/list").
+	globalPluginRouter.Path("/list").
 		HandlerFunc(impl.globalPluginRestHandler.ListAllPlugins).Methods("GET")
 
-	globalPluginRouter.Path("/global/{pluginId}").
+	globalPluginRouter.Path("/{pluginId}").
 		HandlerFunc(impl.globalPluginRestHandler.GetPluginDetailById).Methods("GET")
 }

api/router/router.go

@@ -386,7 +386,7 @@ func (r MuxRouter) Init() {
 	pProfListenerRouter := r.Router.PathPrefix("/orchestrator/debug/pprof").Subrouter()
 	r.pProfRouter.initPProfRouter(pProfListenerRouter)
 
-	globalPluginRouter := r.Router.PathPrefix("/orchestrator/plugin").Subrouter()
+	globalPluginRouter := r.Router.PathPrefix("/orchestrator/plugin/global").Subrouter()
 	r.globalPluginRouter.initGlobalPluginRouter(globalPluginRouter)
 
 	//  deployment router starts

pkg/pipeline/repository/PipelineStageRepository.go

@@ -158,6 +158,7 @@ type PipelineStageRepository interface {
 	GetStepById(stepId int) (*PipelineStageStep, error)
 	MarkStepsDeletedByStageId(stageId int) error
 	MarkStepsDeletedExcludingActiveStepsInUpdateReq(activeStepIdsPresentInReq []int, stageId int) error
+	GetActiveStepsByRefPluginId(refPluginId int) ([]*PipelineStageStep, error)
 
 	CreatePipelineScript(pipelineScript *PluginPipelineScript, tx *pg.Tx) (*PluginPipelineScript, error)
 	UpdatePipelineScript(pipelineScript *PluginPipelineScript) (*PluginPipelineScript, error)
@@ -168,6 +169,7 @@ type PipelineStageRepository interface {
 
 	MarkScriptMappingDeletedByScriptId(scriptId int) error
 	CreateScriptMapping(mappings []ScriptPathArgPortMapping, tx *pg.Tx) error
+	UpdateScriptMapping(mappings []*ScriptPathArgPortMapping, tx *pg.Tx) error
 	GetScriptMappingIdsByStageId(stageId int) ([]int, error)
 	MarkPipelineScriptMappingsDeletedByIds(ids []int, updatedBy int32, tx *pg.Tx) error
 	GetScriptMappingDetailByScriptId(scriptId int) ([]*ScriptPathArgPortMapping, error)
@@ -431,6 +433,18 @@ func (impl *PipelineStageRepositoryImpl) MarkStepsDeletedExcludingActiveStepsInU
 	return nil
 }
 
+func (impl *PipelineStageRepositoryImpl) GetActiveStepsByRefPluginId(refPluginId int) ([]*PipelineStageStep, error) {
+	var steps []*PipelineStageStep
+	err := impl.dbConnection.Model(&steps).
+		Where("ref_plugin_id = ?", refPluginId).
+		Where("deleted = ?", false).Select()
+	if err != nil {
+		impl.logger.Errorw("err in getting all steps by refPluginId", "err", err, "refPluginId", refPluginId)
+		return nil, err
+	}
+	return steps, nil
+}
+
 func (impl *PipelineStageRepositoryImpl) CreatePipelineScript(pipelineScript *PluginPipelineScript, tx *pg.Tx) (*PluginPipelineScript, error) {
 	var err error
 	if tx != nil {
@@ -530,6 +544,27 @@ func (impl *PipelineStageRepositoryImpl) CreateScriptMapping(mappings []ScriptPa
 	return nil
 }
 
+func (impl *PipelineStageRepositoryImpl) UpdateScriptMapping(mappings []*ScriptPathArgPortMapping, tx *pg.Tx) error {
+	var err error
+	if tx != nil {
+		for _, entry := range mappings {
+			err = tx.Update(entry)
+			if err != nil {
+				impl.logger.Errorw("error in updating ScriptPathArgPortMapping", "entry", entry, "err", err)
+				return err
+			}
+		}
+
+	} else {
+		err = impl.dbConnection.Update(&mappings)
+	}
+	if err != nil {
+		impl.logger.Errorw("error in updating pipeline script mappings", "err", err, "mappings", mappings)
+		return err
+	}
+	return nil
+}
+
 func (impl *PipelineStageRepositoryImpl) GetScriptMappingIdsByStageId(stageId int) ([]int, error) {
 	var ids []int
 	query := "SELECT spapm.id from script_path_arg_port_mapping spapm INNER JOIN plugin_pipeline_script pps ON pps.id = spapm.script_id " +