diff --git a/Wire.go b/Wire.go index 6cc0dcab9b..797ab714d2 100644 --- a/Wire.go +++ b/Wire.go @@ -652,6 +652,11 @@ func InitializeApp() (*App, error) { wire.Bind(new(router.SsoLoginRouter), new(*router.SsoLoginRouterImpl)), restHandler.NewSsoLoginRestHandlerImpl, wire.Bind(new(restHandler.SsoLoginRestHandler), new(*restHandler.SsoLoginRestHandlerImpl)), + + router.NewTelemetryRouterImpl, + wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), + restHandler.NewTelemetryRestHandlerImpl, + wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)), telemetry.NewPosthogClient, telemetry.NewTelemetryEventClientImpl, diff --git a/api/restHandler/TelemetryRestHandler.go b/api/restHandler/TelemetryRestHandler.go new file mode 100644 index 0000000000..578407da6a --- /dev/null +++ b/api/restHandler/TelemetryRestHandler.go @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 Devtron Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package restHandler + +import ( + "github.com/devtron-labs/devtron/client/telemetry" + "go.uber.org/zap" + "net/http" +) + +type TelemetryRestHandler interface { + GetClientPlatformIdAndTelemetryUrl(w http.ResponseWriter, r *http.Request) +} + +type TelemetryRestHandlerImpl struct { + logger *zap.SugaredLogger + telemetryEventClient telemetry.TelemetryEventClient +} + +func NewTelemetryRestHandlerImpl(logger *zap.SugaredLogger, + telemetryEventClient telemetry.TelemetryEventClient) *TelemetryRestHandlerImpl { + handler := &TelemetryRestHandlerImpl{logger: logger, telemetryEventClient: telemetryEventClient} + return handler +} + +func (handler TelemetryRestHandlerImpl) GetClientPlatformIdAndTelemetryUrl(w http.ResponseWriter, r *http.Request) { + res, err := handler.telemetryEventClient.GetClientPlatformIdAndTelemetryUrl() + if err != nil { + handler.logger.Errorw("service err, GetClientPlatformIdAndTelemetryUrl", "err", err) + writeJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + writeJsonResp(w, nil, res, http.StatusOK) +} diff --git a/api/router/TelemetryRouter.go b/api/router/TelemetryRouter.go new file mode 100644 index 0000000000..691b7ea082 --- /dev/null +++ b/api/router/TelemetryRouter.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2020 Devtron Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package router + +import ( + "github.com/devtron-labs/devtron/api/restHandler" + "github.com/gorilla/mux" + "go.uber.org/zap" +) + +type TelemetryRouter interface { + initTelemetryRouter(router *mux.Router) +} + +type TelemetryRouterImpl struct { + logger *zap.SugaredLogger + handler restHandler.TelemetryRestHandler +} + +func NewTelemetryRouterImpl(logger *zap.SugaredLogger, handler restHandler.TelemetryRestHandler) *TelemetryRouterImpl { + router := &TelemetryRouterImpl{ + handler: handler, + } + return router +} + +func (router TelemetryRouterImpl) initTelemetryRouter(telemetryRouter *mux.Router) { + telemetryRouter.Path("/ucid"). + HandlerFunc(router.handler.GetClientPlatformIdAndTelemetryUrl).Methods("GET") +} diff --git a/api/router/router.go b/api/router/router.go index 942a0ad4dc..e9b6ac665c 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -73,6 +73,7 @@ type MuxRouter struct { commonRouter CommonRouter grafanaRouter GrafanaRouter ssoLoginRouter SsoLoginRouter + telemetryRouter TelemetryRouter telemetryWatcher telemetry.TelemetryEventClient } @@ -92,7 +93,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter HelmRouter, PipelineConf ReleaseMetricsRouter ReleaseMetricsRouter, deploymentGroupRouter DeploymentGroupRouter, batchOperationRouter BatchOperationRouter, chartGroupRouter ChartGroupRouter, testSuitRouter TestSuitRouter, imageScanRouter ImageScanRouter, policyRouter PolicyRouter, gitOpsConfigRouter GitOpsConfigRouter, dashboardRouter DashboardRouter, attributesRouter AttributesRouter, - commonRouter CommonRouter, grafanaRouter GrafanaRouter, ssoLoginRouter SsoLoginRouter, telemetryWatcher telemetry.TelemetryEventClient) *MuxRouter { + commonRouter CommonRouter, grafanaRouter GrafanaRouter, ssoLoginRouter SsoLoginRouter, telemetryRouter TelemetryRouter, telemetryWatcher telemetry.TelemetryEventClient) *MuxRouter { r := &MuxRouter{ Router: mux.NewRouter(), HelmRouter: HelmRouter, @@ -136,6 +137,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter HelmRouter, PipelineConf commonRouter: commonRouter, grafanaRouter: grafanaRouter, ssoLoginRouter: ssoLoginRouter, + telemetryRouter: telemetryRouter, telemetryWatcher: telemetryWatcher, } return r @@ -259,4 +261,7 @@ func (r MuxRouter) Init() { ssoLoginRouter := r.Router.PathPrefix("/orchestrator/sso").Subrouter() r.ssoLoginRouter.initSsoLoginRouter(ssoLoginRouter) + + telemetryRouter := r.Router.PathPrefix("/orchestrator/telemetry").Subrouter() + r.telemetryRouter.initTelemetryRouter(telemetryRouter) } diff --git a/client/telemetry/PosthogClient.go b/client/telemetry/PosthogClient.go index 3258ac43d2..74abd9c148 100644 --- a/client/telemetry/PosthogClient.go +++ b/client/telemetry/PosthogClient.go @@ -19,12 +19,15 @@ package telemetry import ( "github.com/caarlos0/env" + "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "go.uber.org/zap" + "time" ) type PosthogClient struct { Client posthog.Client + cache *cache.Cache } type PosthogConfig struct { @@ -32,6 +35,7 @@ type PosthogConfig struct { PosthogEndpoint string `env:"POSTHOG_ENDPOINT" envDefault:"https://app.posthog.com"` SummaryInterval int `env:"SUMMARY_INTERVAL" envDefault:"24"` HeartbeatInterval int `env:"HEARTBEAT_INTERVAL" envDefault:"3"` + CacheExpiry int `env:"CACHE_EXPIRY" envDefault:"120"` } func GetPosthogConfig() (*PosthogConfig, error) { @@ -52,8 +56,11 @@ func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { } client, _ := posthog.NewWithConfig(cfg.ApiKey, posthog.Config{Endpoint: cfg.PosthogEndpoint}) //defer client.Close() + d := time.Duration(cfg.CacheExpiry) + c := cache.New(d*time.Minute, 240*time.Minute) pgClient := &PosthogClient{ Client: client, + cache: c, } return pgClient, nil } diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index cb9c98742b..d415a9be7c 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -3,7 +3,6 @@ package telemetry import ( "encoding/json" "fmt" - client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" util2 "github.com/devtron-labs/devtron/internal/util" @@ -11,6 +10,7 @@ import ( "github.com/devtron-labs/devtron/pkg/user" "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" + "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -29,7 +29,6 @@ type TelemetryEventClientImpl struct { clusterService cluster.ClusterService K8sUtil *util2.K8sUtil aCDAuthConfig *user.ACDAuthConfig - config *client.EventClientConfig environmentService cluster.EnvironmentService userService user.UserService appListingRepository repository.AppListingRepository @@ -40,10 +39,11 @@ type TelemetryEventClientImpl struct { } type TelemetryEventClient interface { + GetClientPlatformIdAndTelemetryUrl() (*PosthogData, error) } func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, - K8sUtil *util2.K8sUtil, aCDAuthConfig *user.ACDAuthConfig, config *client.EventClientConfig, + K8sUtil *util2.K8sUtil, aCDAuthConfig *user.ACDAuthConfig, environmentService cluster.EnvironmentService, userService user.UserService, appListingRepository repository.AppListingRepository, PosthogClient *PosthogClient, ciPipelineRepository pipelineConfig.CiPipelineRepository, pipelineRepository pipelineConfig.PipelineRepository, @@ -55,12 +55,13 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, cron: cron, logger: logger, client: client, clusterService: clusterService, - K8sUtil: K8sUtil, aCDAuthConfig: aCDAuthConfig, config: config, + K8sUtil: K8sUtil, aCDAuthConfig: aCDAuthConfig, environmentService: environmentService, userService: userService, appListingRepository: appListingRepository, PosthogClient: PosthogClient, ciPipelineRepository: ciPipelineRepository, pipelineRepository: pipelineRepository, posthogConfig: posthogConfig, } + watcher.HeartbeatEventForTelemetry() _, err := cron.AddFunc(fmt.Sprintf("@every %dm", watcher.posthogConfig.SummaryInterval), watcher.SummaryEventForTelemetry) if err != nil { @@ -122,30 +123,11 @@ func (d TelemetryEventType) String() string { } func (impl *TelemetryEventClientImpl) SummaryEventForTelemetry() { - client, err := impl.K8sUtil.GetClientForInCluster() + ucid, err := impl.getUCID() if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) return } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) - if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { - // if not found, create new cm - cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} - data := map[string]string{} - data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number - cm.Data = data - _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) - if err != nil { - return - } - } - if cm == nil { - impl.logger.Errorw("cm found nil inside telemetry summary event", "cm", cm) - return - } - dataMap := cm.Data - ucid := dataMap[DevtronUniqueClientIdConfigMapKey] - discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster() if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) @@ -229,31 +211,11 @@ func (impl *TelemetryEventClientImpl) SummaryEventForTelemetry() { } func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { - client, err := impl.K8sUtil.GetClientForInCluster() + ucid, err := impl.getUCID() if err != nil { impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err) return } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) - if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { - // if not found, create new cm - cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} - data := map[string]string{} - data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number - cm.Data = data - _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) - if err != nil { - impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err) - return - } - } - if cm == nil { - impl.logger.Errorw("configmap found nil for telemetry heartbeat event", "cm", cm) - return - } - dataMap := cm.Data - ucid := dataMap[DevtronUniqueClientIdConfigMapKey] - discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster() if err != nil { impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err) @@ -283,3 +245,56 @@ func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { Properties: prop, }) } + +func (impl *TelemetryEventClientImpl) GetClientPlatformIdAndTelemetryUrl() (*PosthogData, error) { + ucid, err := impl.getUCID() + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return nil, err + } + data := &PosthogData{ + Url: impl.posthogConfig.PosthogEndpoint, + UCID: ucid, + } + return data, err +} + +type PosthogData struct { + Url string `json:"url,omitempty"` + UCID string `json:"ucid,omitempty"` +} + +func (impl *TelemetryEventClientImpl) getUCID() (string, error) { + ucid, found := impl.PosthogClient.cache.Get(DevtronUniqueClientIdConfigMapKey) + if found { + return ucid.(string), nil + } else { + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", err + } + + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { + // if not found, create new cm + cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} + data := map[string]string{} + data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number + cm.Data = data + _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", err + } + } + dataMap := cm.Data + ucid = dataMap[DevtronUniqueClientIdConfigMapKey] + impl.PosthogClient.cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, cache.DefaultExpiration) + if cm == nil { + impl.logger.Errorw("configmap not found while getting unique client id", "cm", cm) + return ucid.(string), err + } + } + return ucid.(string), nil +} diff --git a/pkg/sso/SSOLoginService.go b/pkg/sso/SSOLoginService.go index bd02ee2d57..4264bc0c28 100644 --- a/pkg/sso/SSOLoginService.go +++ b/pkg/sso/SSOLoginService.go @@ -23,6 +23,7 @@ import ( "github.com/argoproj/argo-cd/util/session" "github.com/devtron-labs/devtron/api/bean" session2 "github.com/devtron-labs/devtron/client/argocdServer/session" + "github.com/devtron-labs/devtron/client/telemetry" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/cluster" @@ -53,13 +54,14 @@ type SSOLoginServiceImpl struct { clusterService cluster.ClusterService envService cluster.EnvironmentService aCDAuthConfig *user.ACDAuthConfig + posthogConfig *telemetry.PosthogConfig } func NewSSOLoginServiceImpl(userAuthRepository repository.UserAuthRepository, sessionManager *session.SessionManager, client session2.ServiceClient, logger *zap.SugaredLogger, userRepository repository.UserRepository, userGroupRepository repository.RoleGroupRepository, ssoLoginRepository repository.SSOLoginRepository, K8sUtil *util.K8sUtil, clusterService cluster.ClusterService, envService cluster.EnvironmentService, - aCDAuthConfig *user.ACDAuthConfig) *SSOLoginServiceImpl { + aCDAuthConfig *user.ACDAuthConfig, posthogConfig *telemetry.PosthogConfig) *SSOLoginServiceImpl { serviceImpl := &SSOLoginServiceImpl{ userAuthRepository: userAuthRepository, sessionManager: sessionManager, @@ -72,6 +74,7 @@ func NewSSOLoginServiceImpl(userAuthRepository repository.UserAuthRepository, se clusterService: clusterService, envService: envService, aCDAuthConfig: aCDAuthConfig, + posthogConfig: posthogConfig, } return serviceImpl } diff --git a/wire_gen.go b/wire_gen.go index 9c5baafa8e..1d1efdc7a4 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -279,7 +279,11 @@ func InitializeApp() (*App, error) { pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(natsPublishClientImpl, sugaredLogger, cdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) ssoLoginRepositoryImpl := repository.NewSSOLoginRepositoryImpl(db) - ssoLoginServiceImpl := sso.NewSSOLoginServiceImpl(userAuthRepositoryImpl, sessionManager, sessionServiceClientImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, ssoLoginRepositoryImpl, k8sUtil, clusterServiceImpl, environmentServiceImpl, acdAuthConfig) + posthogConfig, err := telemetry.GetPosthogConfig() + if err != nil { + return nil, err + } + ssoLoginServiceImpl := sso.NewSSOLoginServiceImpl(userAuthRepositoryImpl, sessionManager, sessionServiceClientImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, ssoLoginRepositoryImpl, k8sUtil, clusterServiceImpl, environmentServiceImpl, acdAuthConfig, posthogConfig) userAuthHandlerImpl := restHandler.NewUserAuthHandlerImpl(userAuthServiceImpl, validate, sugaredLogger, enforcerImpl, pubSubClient, userServiceImpl, ssoLoginServiceImpl) userAuthRouterImpl := router.NewUserAuthRouterImpl(sugaredLogger, userAuthHandlerImpl, argocdServerConfig, dexConfig, argoCDSettings, userServiceImpl) pumpImpl := connector.NewPumpImpl(sugaredLogger) @@ -394,15 +398,13 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - posthogConfig, err := telemetry.GetPosthogConfig() - if err != nil { - return nil, err - } - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sUtil, acdAuthConfig, eventClientConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, posthogConfig) + telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sUtil, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, posthogConfig) if err != nil { return nil, err } - muxRouter := router.NewMuxRouter(sugaredLogger, helmRouterImpl, pipelineConfigRouterImpl, migrateDbRouterImpl, clusterAccountsRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, clusterHelmConfigRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, projectManagementRouterImpl, gitProviderRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusUpdateHandlerImpl, ciEventHandlerImpl, pubSubClient, userRouterImpl, cronBasedEventReceiverImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, testSuitRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryEventClientImpl) + telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImpl) + telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, helmRouterImpl, pipelineConfigRouterImpl, migrateDbRouterImpl, clusterAccountsRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, clusterHelmConfigRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, projectManagementRouterImpl, gitProviderRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusUpdateHandlerImpl, ciEventHandlerImpl, pubSubClient, userRouterImpl, cronBasedEventReceiverImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, testSuitRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImpl) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, sessionManager, versionServiceImpl, enforcer, db, pubSubClient) return mainApp, nil }