From c3987b802a0fa3fd8ca0cb03cf872c06b12031ea Mon Sep 17 00:00:00 2001 From: ashishdevtron <141303172+ashishdevtron@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:05:34 +0530 Subject: [PATCH] feat: added posthog events for cloud provider and version (#4443) * added cloud provider event * added logger for testing * logger removed * common-lib version upgraded --- Wire.go | 4 + client/telemetry/TelemetryEventClient.go | 117 ++++++++++++++---- .../telemetry/TelemetryEventClientExtended.go | 45 ++++--- cmd/external-app/wire.go | 3 + cmd/external-app/wire_gen.go | 4 +- go.mod | 2 +- go.sum | 4 +- .../ProviderIdentifierService.go | 70 +++++++++++ .../cloud-provider-identifier/bean/bean.go | 41 ++++++ .../providers/alibaba.go | 78 ++++++++++++ .../providers/aws.go | 93 ++++++++++++++ .../providers/azure.go | 48 +++++++ .../providers/digitalOcean.go | 69 +++++++++++ .../providers/google.go | 48 +++++++ .../providers/oracle.go | 84 +++++++++++++ .../common-lib/utils/k8s/K8sUtil.go | 17 +++ vendor/modules.txt | 5 +- wire_gen.go | 4 +- 18 files changed, 689 insertions(+), 47 deletions(-) create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/ProviderIdentifierService.go create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/bean/bean.go create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/alibaba.go create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/aws.go create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/azure.go create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/digitalOcean.go create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/google.go create mode 100644 vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/oracle.go diff --git a/Wire.go b/Wire.go index ebc1562d6bd..c3a7b568989 100644 --- a/Wire.go +++ b/Wire.go @@ -22,6 +22,7 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" + cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" pubsub1 "github.com/devtron-labs/common-lib/pubsub-lib" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" @@ -714,6 +715,9 @@ func InitializeApp() (*App, error) { wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)), telemetry.NewPosthogClient, + cloudProviderIdentifier.NewProviderIdentifierServiceImpl, + wire.Bind(new(cloudProviderIdentifier.ProviderIdentifierService), new(*cloudProviderIdentifier.ProviderIdentifierServiceImpl)), + telemetry.NewTelemetryEventClientImplExtended, wire.Bind(new(telemetry.TelemetryEventClient), new(*telemetry.TelemetryEventClientImplExtended)), diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index d5bbe8c68ea..c806f42f7ff 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" "net/http" "time" @@ -40,22 +41,23 @@ const SKIPPED_ONBOARDING_CONST = "SkippedOnboarding" const ADMIN_EMAIL_ID_CONST = "admin" type TelemetryEventClientImpl struct { - cron *cron.Cron - logger *zap.SugaredLogger - client *http.Client - clusterService cluster.ClusterService - K8sUtil *k8s.K8sServiceImpl - aCDAuthConfig *util3.ACDAuthConfig - userService user2.UserService - attributeRepo repository.AttributesRepository - ssoLoginService sso.SSOLoginService - PosthogClient *PosthogClient - moduleRepository moduleRepo.ModuleRepository - serverDataStore *serverDataStore.ServerDataStore - userAuditService user2.UserAuditService - helmAppClient client.HelmAppClient - InstalledAppRepository repository2.InstalledAppRepository - userAttributesRepository repository.UserAttributesRepository + cron *cron.Cron + logger *zap.SugaredLogger + client *http.Client + clusterService cluster.ClusterService + K8sUtil *k8s.K8sServiceImpl + aCDAuthConfig *util3.ACDAuthConfig + userService user2.UserService + attributeRepo repository.AttributesRepository + ssoLoginService sso.SSOLoginService + PosthogClient *PosthogClient + moduleRepository moduleRepo.ModuleRepository + serverDataStore *serverDataStore.ServerDataStore + userAuditService user2.UserAuditService + helmAppClient client.HelmAppClient + InstalledAppRepository repository2.InstalledAppRepository + userAttributesRepository repository.UserAttributesRepository + cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService } type TelemetryEventClient interface { @@ -70,7 +72,8 @@ type TelemetryEventClient interface { func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, - PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository) (*TelemetryEventClientImpl, error) { + PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, + cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService) (*TelemetryEventClientImpl, error) { cron := cron.New( cron.WithChain()) cron.Start() @@ -80,13 +83,14 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, client: client, clusterService: clusterService, K8sUtil: K8sUtil, aCDAuthConfig: aCDAuthConfig, userService: userService, attributeRepo: attributeRepo, - ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, - moduleRepository: moduleRepository, - serverDataStore: serverDataStore, - userAuditService: userAuditService, - helmAppClient: helmAppClient, - InstalledAppRepository: InstalledAppRepository, + ssoLoginService: ssoLoginService, + PosthogClient: PosthogClient, + moduleRepository: moduleRepository, + serverDataStore: serverDataStore, + userAuditService: userAuditService, + helmAppClient: helmAppClient, + InstalledAppRepository: InstalledAppRepository, + cloudProviderIdentifierService: cloudProviderIdentifierService, } watcher.HeartbeatEventForTelemetry() @@ -132,6 +136,7 @@ type TelemetryEventEA struct { SkippedOnboarding bool `json:"SkippedOnboarding"` HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"` + ClusterProvider string `json:"clusterProvider,omitempty"` } const DevtronUniqueClientIdConfigMap = "devtron-ucid" @@ -315,6 +320,13 @@ func (impl *TelemetryEventClientImpl) SendSummaryEvent(eventType string) error { payload.HelmChartSuccessfulDeploymentCount = helmChartSuccessfulDeploymentCount payload.ExternalHelmAppClusterCount = ExternalHelmAppClusterCount + provider, err := impl.cloudProviderIdentifierService.IdentifyProvider() + if err != nil { + impl.logger.Errorw("exception while getting cluster provider", "error", err) + return err + } + payload.ClusterProvider = provider + latestUser, err := impl.userAuditService.GetLatestUser() if err == nil { loginTime := latestUser.UpdatedOn @@ -459,8 +471,27 @@ func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryE return nil, err } + discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster() + if err != nil { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return nil, err + } + k8sServerVersion, err := discoveryClient.ServerVersion() + if err != nil { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return nil, err + } + payload := &TelemetryEventEA{UCID: ucid, Timestamp: time.Now(), EventType: InstallationSuccess, DevtronVersion: "v1"} payload.DevtronMode = util.GetDevtronVersion().ServerMode + payload.ServerVersion = k8sServerVersion.String() + + provider, err := impl.cloudProviderIdentifierService.IdentifyProvider() + if err != nil { + impl.logger.Errorw("exception while getting cluster provider", "error", err) + return nil, err + } + payload.ClusterProvider = provider reqBody, err := json.Marshal(payload) if err != nil { @@ -507,8 +538,27 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error return err } + discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster() + if err != nil { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } + k8sServerVersion, err := discoveryClient.ServerVersion() + if err != nil { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } + payload := &TelemetryEventEA{UCID: ucid, Timestamp: time.Now(), EventType: DashboardAccessed, DevtronVersion: "v1"} payload.DevtronMode = util.GetDevtronVersion().ServerMode + payload.ServerVersion = k8sServerVersion.String() + + provider, err := impl.cloudProviderIdentifierService.IdentifyProvider() + if err != nil { + impl.logger.Errorw("exception while getting cluster provider", "error", err) + return err + } + payload.ClusterProvider = provider reqBody, err := json.Marshal(payload) if err != nil { @@ -555,8 +605,27 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro return err } + discoveryClient, err := impl.K8sUtil.GetK8sDiscoveryClientInCluster() + if err != nil { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } + k8sServerVersion, err := discoveryClient.ServerVersion() + if err != nil { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } + payload := &TelemetryEventEA{UCID: ucid, Timestamp: time.Now(), EventType: DashboardLoggedIn, DevtronVersion: "v1"} payload.DevtronMode = util.GetDevtronVersion().ServerMode + payload.ServerVersion = k8sServerVersion.String() + + provider, err := impl.cloudProviderIdentifierService.IdentifyProvider() + if err != nil { + impl.logger.Errorw("exception while getting cluster provider", "error", err) + return err + } + payload.ClusterProvider = provider reqBody, err := json.Marshal(payload) if err != nil { diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 77e5acc219b..cdf27e02b8b 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -2,6 +2,7 @@ package telemetry import ( "encoding/json" + cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" "net/http" "time" @@ -59,7 +60,8 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http materialRepository pipelineConfig.MaterialRepository, ciTemplateRepository pipelineConfig.CiTemplateRepository, chartRepository chartRepoRepository.ChartRepository, userAuditService user2.UserAuditService, ciBuildConfigService pipeline.CiBuildConfigService, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, - helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, userAttributesRepository repository.UserAttributesRepository) (*TelemetryEventClientImplExtended, error) { + helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, userAttributesRepository repository.UserAttributesRepository, + cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService) (*TelemetryEventClientImplExtended, error) { cron := cron.New( cron.WithChain()) @@ -81,22 +83,23 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http ciBuildConfigService: ciBuildConfigService, TelemetryEventClientImpl: &TelemetryEventClientImpl{ - cron: cron, - logger: logger, - client: client, - clusterService: clusterService, - K8sUtil: K8sUtil, - aCDAuthConfig: aCDAuthConfig, - userService: userService, - attributeRepo: attributeRepo, - ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, - moduleRepository: moduleRepository, - serverDataStore: serverDataStore, - userAuditService: userAuditService, - helmAppClient: helmAppClient, - InstalledAppRepository: InstalledAppRepository, - userAttributesRepository: userAttributesRepository, + cron: cron, + logger: logger, + client: client, + clusterService: clusterService, + K8sUtil: K8sUtil, + aCDAuthConfig: aCDAuthConfig, + userService: userService, + attributeRepo: attributeRepo, + ssoLoginService: ssoLoginService, + PosthogClient: PosthogClient, + moduleRepository: moduleRepository, + serverDataStore: serverDataStore, + userAuditService: userAuditService, + helmAppClient: helmAppClient, + InstalledAppRepository: InstalledAppRepository, + userAttributesRepository: userAttributesRepository, + cloudProviderIdentifierService: cloudProviderIdentifierService, }, } @@ -167,6 +170,7 @@ type TelemetryEventDto struct { HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"` + ClusterProvider string `json:"clusterProvider,omitempty"` } func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { @@ -322,6 +326,13 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) payload.HelmChartSuccessfulDeploymentCount = HelmChartSuccessfulDeploymentCount payload.ExternalHelmAppClusterCount = ExternalHelmAppClusterCount + provider, err := impl.cloudProviderIdentifierService.IdentifyProvider() + if err != nil { + impl.logger.Errorw("exception while getting cluster provider", "error", err) + return err + } + payload.ClusterProvider = provider + latestUser, err := impl.userAuditService.GetLatestUser() if err == nil { loginTime := latestUser.UpdatedOn diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 044f9c0d984..8e93545d3cc 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -5,6 +5,7 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" + cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" chartProvider "github.com/devtron-labs/devtron/api/appStore/chartProvider" @@ -114,6 +115,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(session.ServiceClient), new(*middleware.LoginService)), connector.NewPumpImpl, wire.Bind(new(connector.Pump), new(*connector.PumpImpl)), + cloudProviderIdentifier.NewProviderIdentifierServiceImpl, + wire.Bind(new(cloudProviderIdentifier.ProviderIdentifierService), new(*cloudProviderIdentifier.ProviderIdentifierServiceImpl)), telemetry.NewTelemetryEventClientImpl, wire.Bind(new(telemetry.TelemetryEventClient), new(*telemetry.TelemetryEventClientImpl)), diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index c95ee93e359..ef258c929a0 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -10,6 +10,7 @@ import ( "github.com/devtron-labs/authenticator/apiToken" "github.com/devtron-labs/authenticator/client" "github.com/devtron-labs/authenticator/middleware" + "github.com/devtron-labs/common-lib/cloud-provider-identifier" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" chartProvider2 "github.com/devtron-labs/devtron/api/appStore/chartProvider" @@ -299,7 +300,8 @@ func InitializeApp() (*App, error) { return nil, err } moduleRepositoryImpl := moduleRepo.NewModuleRepositoryImpl(db) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sUtil, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl) + providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) + telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sUtil, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl, providerIdentifierServiceImpl) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index d59b4fcb9af..4da1b2bec8f 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.33 - github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e + github.com/devtron-labs/common-lib v0.0.10 github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index fd9dff6c690..043c296b162 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.33 h1:FpAV3ZgFluaRFcMwPpwxr/mwSipJ16XRvgABq3BzP5Y= github.com/devtron-labs/authenticator v0.4.33/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e h1:tPD2kWowGhgH3g5TRZMKgrvuNAgpFE6E6sxZaTAHNSU= -github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= +github.com/devtron-labs/common-lib v0.0.10 h1:3ayOUwIedXTvBEj80mKJ0iLo06glZ9nzY/WFsYFpYGM= +github.com/devtron-labs/common-lib v0.0.10/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/ProviderIdentifierService.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/ProviderIdentifierService.go new file mode 100644 index 00000000000..16c1ad80db1 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/ProviderIdentifierService.go @@ -0,0 +1,70 @@ +package providerIdentifier + +import ( + "github.com/devtron-labs/common-lib/cloud-provider-identifier/bean" + "github.com/devtron-labs/common-lib/cloud-provider-identifier/providers" + "go.uber.org/zap" +) + +type Identifier interface { + Identify() (string, error) + IdentifyViaMetadataServer(detected chan<- string) +} + +type ProviderIdentifierService interface { + IdentifyProvider() (string, error) +} + +type ProviderIdentifierServiceImpl struct { + logger *zap.SugaredLogger +} + +func NewProviderIdentifierServiceImpl( + logger *zap.SugaredLogger) *ProviderIdentifierServiceImpl { + providerIdentifierService := &ProviderIdentifierServiceImpl{ + logger: logger, + } + return providerIdentifierService +} + +func (impl *ProviderIdentifierServiceImpl) IdentifyProvider() (string, error) { + identifiers := []Identifier{ + &providers.IdentifyAlibaba{Logger: impl.logger}, + &providers.IdentifyAmazon{Logger: impl.logger}, + &providers.IdentifyAzure{Logger: impl.logger}, + &providers.IdentifyDigitalOcean{Logger: impl.logger}, + &providers.IdentifyGoogle{Logger: impl.logger}, + &providers.IdentifyOracle{Logger: impl.logger}, + } + + identifiedProv := bean.Unknown + var err error + for _, prov := range identifiers { + identifiedProv, err = prov.Identify() + if err != nil { + continue + } + if identifiedProv != bean.Unknown { + return identifiedProv, nil + } + } + + detected := make(chan string) + defer close(detected) + + provs := make([]func(chan<- string), 0) + for _, prov := range identifiers { + provs = append(provs, prov.IdentifyViaMetadataServer) + } + + for _, functions := range provs { + go functions(detected) + } + for range provs { + d := <-detected + if d != bean.Unknown { + return d, nil + } + } + return bean.Unknown, nil +} diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/bean/bean.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/bean/bean.go new file mode 100644 index 00000000000..87e05c45e6f --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/bean/bean.go @@ -0,0 +1,41 @@ +package bean + +const ( + Amazon = "amazon" + Alibaba = "alibaba" + Azure = "azure" + Google = "google" + Oracle = "oracle" + DigitalOcean = "digitalocean" + Unknown = "unknown" +) + +const ( + AlibabaIdentifierString = "Alibaba Cloud" + AmazonIdentifierString = "amazon" + AzureIdentifierString = "Microsoft Corporation" + DigitalOceanIdentifierString = "DigitalOcean" + GoogleIdentifierString = "Google" + OracleIdentifierString = "OracleCloud" +) + +const ( + AlibabaSysFile = "/sys/class/dmi/id/product_name" + AmazonSysFile = "/sys/class/dmi/id/product_version" + AzureSysFile = "/sys/class/dmi/id/sys_vendor" + DigitalOceanSysFile = "/sys/class/dmi/id/sys_vendor" + GoogleSysFile = "/sys/class/dmi/id/product_name" + OracleSysFile = "/sys/class/dmi/id/chassis_asset_tag" +) + +const ( + AlibabaMetadataServer = "http://100.100.100.200/latest/meta-data/instance/instance-type" + TokenForAlibabaMetadataServer = "http://100.100.100.200/latest/api/token" + AmazonMetadataServer = "http://169.254.169.254/latest/dynamic/instance-identity/document" + TokenForAmazonMetadataServerV2 = "http://169.254.169.254/latest/api/token" + AzureMetadataServer = "http://169.254.169.254/metadata/instance?api-version=2021-02-01" + DigitalOceanMetadataServer = "http://169.254.169.254/metadata/v1.json" + GoogleMetadataServer = "http://metadata.google.internal/computeMetadata/v1/instance/tags" + OracleMetadataServerV1 = "http://169.254.169.254/opc/v1/instance/metadata/" + OracleMetadataServerV2 = "http://169.254.169.254/opc/v2/instance/metadata/" +) diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/alibaba.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/alibaba.go new file mode 100644 index 00000000000..af2e1947331 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/alibaba.go @@ -0,0 +1,78 @@ +package providers + +import ( + "io" + "net/http" + "os" + "strings" + + "github.com/devtron-labs/common-lib/cloud-provider-identifier/bean" + "go.uber.org/zap" +) + +type IdentifyAlibaba struct { + Logger *zap.SugaredLogger +} + +func (impl *IdentifyAlibaba) Identify() (string, error) { + data, err := os.ReadFile(bean.AlibabaSysFile) + if err != nil { + impl.Logger.Errorw("error while reading file", "error", err) + return bean.Unknown, err + } + if strings.Contains(string(data), bean.AlibabaIdentifierString) { + return bean.Alibaba, nil + } + return bean.Unknown, nil +} + +func (impl *IdentifyAlibaba) IdentifyViaMetadataServer(detected chan<- string) { + req, err := http.NewRequest("PUT", bean.TokenForAlibabaMetadataServer, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + req.Header.Set("X-aliyun-ecs-metadata-token-ttl-seconds", "21600") + tokenResp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + defer tokenResp.Body.Close() + token, err := io.ReadAll(tokenResp.Body) + if err != nil { + impl.Logger.Errorw("error while reading response body", "error", err, "respBody", tokenResp.Body) + detected <- bean.Unknown + return + } + req, err = http.NewRequest("GET", bean.AlibabaMetadataServer, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + req.Header.Set("X-aliyun-ecs-metadata-token", string(token)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + if resp.StatusCode == http.StatusOK { + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + impl.Logger.Errorw("error while reading response body", "error", err, "respBody", resp.Body) + detected <- bean.Unknown + return + } + if strings.HasPrefix(string(body), "ecs.") { + detected <- bean.Alibaba + } + } else { + detected <- bean.Unknown + return + } +} diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/aws.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/aws.go new file mode 100644 index 00000000000..28df250e7ed --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/aws.go @@ -0,0 +1,93 @@ +package providers + +import ( + "encoding/json" + "io" + "net/http" + "os" + "strings" + + "github.com/devtron-labs/common-lib/cloud-provider-identifier/bean" + "go.uber.org/zap" +) + +type instanceIdentityResponse struct { + ImageID string `json:"imageId"` + InstanceID string `json:"instanceId"` +} + +type IdentifyAmazon struct { + Logger *zap.SugaredLogger +} + +func (impl *IdentifyAmazon) Identify() (string, error) { + data, err := os.ReadFile(bean.AmazonSysFile) + if err != nil { + impl.Logger.Errorw("error while reading file", "error", err) + return bean.Unknown, err + } + if strings.Contains(string(data), bean.AmazonIdentifierString) { + return bean.Amazon, nil + } + return bean.Unknown, nil +} + +func (impl *IdentifyAmazon) IdentifyViaMetadataServer(detected chan<- string) { + r := instanceIdentityResponse{} + req, err := http.NewRequest("PUT", bean.TokenForAmazonMetadataServerV2, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "21600") + tokenResp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + defer tokenResp.Body.Close() + token, err := io.ReadAll(tokenResp.Body) + if err != nil { + impl.Logger.Errorw("error while reading response body", "error", err, "respBody", tokenResp.Body) + detected <- bean.Unknown + return + } + req, err = http.NewRequest("GET", bean.AmazonMetadataServer, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + req.Header.Set("X-aws-ec2-metadata-token", string(token)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + if resp.StatusCode == http.StatusOK { + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + impl.Logger.Errorw("error while reading response body", "error", err, "respBody", resp.Body) + detected <- bean.Unknown + return + } + err = json.Unmarshal(body, &r) + if err != nil { + impl.Logger.Errorw("error while unmarshaling json", "error", err, "body", body) + detected <- bean.Unknown + return + } + if strings.HasPrefix(r.ImageID, "ami-") && + strings.HasPrefix(r.InstanceID, "i-") { + detected <- bean.Amazon + return + } + } else { + detected <- bean.Unknown + return + } +} diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/azure.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/azure.go new file mode 100644 index 00000000000..e7178d80c26 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/azure.go @@ -0,0 +1,48 @@ +package providers + +import ( + "net/http" + "os" + "strings" + + "github.com/devtron-labs/common-lib/cloud-provider-identifier/bean" + "go.uber.org/zap" +) + +type IdentifyAzure struct { + Logger *zap.SugaredLogger +} + +func (impl *IdentifyAzure) Identify() (string, error) { + data, err := os.ReadFile(bean.AzureSysFile) + if err != nil { + impl.Logger.Errorw("error while reading file", "error", err) + return bean.Unknown, err + } + if strings.Contains(string(data), bean.AzureIdentifierString) { + return bean.Azure, nil + } + return bean.Unknown, nil +} + +func (impl *IdentifyAzure) IdentifyViaMetadataServer(detected chan<- string) { + req, err := http.NewRequest("GET", bean.AzureMetadataServer, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + req.Header.Set("Metadata", "true") + resp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + if resp.StatusCode == http.StatusOK { + detected <- bean.Azure + } else { + detected <- bean.Unknown + return + } +} diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/digitalOcean.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/digitalOcean.go new file mode 100644 index 00000000000..19111e4d25b --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/digitalOcean.go @@ -0,0 +1,69 @@ +package providers + +import ( + "encoding/json" + "io" + "net/http" + "os" + "strings" + + "github.com/devtron-labs/common-lib/cloud-provider-identifier/bean" + "go.uber.org/zap" +) + +type digitalOceanMetadataResponse struct { + DropletID int `json:"droplet_id"` +} + +type IdentifyDigitalOcean struct { + Logger *zap.SugaredLogger +} + +func (impl *IdentifyDigitalOcean) Identify() (string, error) { + data, err := os.ReadFile(bean.DigitalOceanSysFile) + if err != nil { + impl.Logger.Errorw("error while reading file", "error", err) + return bean.Unknown, err + } + if strings.Contains(string(data), bean.DigitalOceanIdentifierString) { + return bean.DigitalOcean, nil + } + return bean.Unknown, nil +} + +func (impl *IdentifyDigitalOcean) IdentifyViaMetadataServer(detected chan<- string) { + r := digitalOceanMetadataResponse{} + req, err := http.NewRequest("GET", bean.DigitalOceanMetadataServer, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + if resp.StatusCode == http.StatusOK { + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + impl.Logger.Errorw("error while reading response body", "error", err, "respBody", resp.Body) + detected <- bean.Unknown + return + } + err = json.Unmarshal(body, &r) + if err != nil { + impl.Logger.Errorw("error while unmarshaling json", "error", err, "body", body) + detected <- bean.Unknown + return + } + if r.DropletID > 0 { + detected <- bean.DigitalOcean + } + } else { + detected <- bean.Unknown + return + } +} diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/google.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/google.go new file mode 100644 index 00000000000..0e2437dc970 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/google.go @@ -0,0 +1,48 @@ +package providers + +import ( + "net/http" + "os" + "strings" + + "github.com/devtron-labs/common-lib/cloud-provider-identifier/bean" + "go.uber.org/zap" +) + +type IdentifyGoogle struct { + Logger *zap.SugaredLogger +} + +func (impl *IdentifyGoogle) Identify() (string, error) { + data, err := os.ReadFile(bean.GoogleSysFile) + if err != nil { + impl.Logger.Errorw("error while reading file", "error", err) + return bean.Unknown, err + } + if strings.Contains(string(data), bean.GoogleIdentifierString) { + return bean.Google, nil + } + return bean.Unknown, nil +} + +func (impl *IdentifyGoogle) IdentifyViaMetadataServer(detected chan<- string) { + req, err := http.NewRequest("GET", bean.GoogleMetadataServer, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + req.Header.Set("Metadata-Flavor", "Google") + resp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + if resp.StatusCode == http.StatusOK { + detected <- bean.Google + } else { + detected <- bean.Unknown + return + } +} diff --git a/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/oracle.go b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/oracle.go new file mode 100644 index 00000000000..43b8cca3fb3 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/cloud-provider-identifier/providers/oracle.go @@ -0,0 +1,84 @@ +package providers + +import ( + "encoding/json" + "io" + "net/http" + "os" + "strings" + + "github.com/devtron-labs/common-lib/cloud-provider-identifier/bean" + "go.uber.org/zap" +) + +type oracleMetadataResponse struct { + OkeTM string `json:"oke-tm"` +} + +type IdentifyOracle struct { + Logger *zap.SugaredLogger +} + +func (impl *IdentifyOracle) Identify() (string, error) { + data, err := os.ReadFile(bean.OracleSysFile) + if err != nil { + impl.Logger.Errorw("error while reading file", "error", err) + return bean.Unknown, err + } + if strings.Contains(string(data), bean.OracleIdentifierString) { + return bean.Oracle, nil + } + return bean.Unknown, nil +} + +func (impl *IdentifyOracle) IdentifyViaMetadataServer(detected chan<- string) { + r := oracleMetadataResponse{} + req, err := http.NewRequest("GET", bean.OracleMetadataServerV1, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + if resp.StatusCode == http.StatusNotFound { + req, err = http.NewRequest("GET", bean.OracleMetadataServerV2, nil) + if err != nil { + impl.Logger.Errorw("error while creating new request", "error", err) + detected <- bean.Unknown + return + } + req.Header.Set("Authorization", "Bearer Oracle") + resp, err = http.DefaultClient.Do(req) + if err != nil { + impl.Logger.Errorw("error while requesting", "error", err, "request", req) + detected <- bean.Unknown + return + } + } + if resp.StatusCode == http.StatusOK { + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + impl.Logger.Errorw("error while reading response body", "error", err, "respBody", resp.Body) + detected <- bean.Unknown + return + } + err = json.Unmarshal(body, &r) + if err != nil { + impl.Logger.Errorw("error while unmarshaling json", "error", err, "body", body) + detected <- bean.Unknown + return + } + if strings.Contains(r.OkeTM, "oke") { + detected <- bean.Oracle + } + } else { + detected <- bean.Unknown + return + } +} diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go index 798aeb92e65..751ea3bcea1 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go @@ -118,6 +118,23 @@ type K8sService interface { GetClientForInCluster() (*v12.CoreV1Client, error) GetCoreV1Client(clusterConfig *ClusterConfig) (*v12.CoreV1Client, error) GetRestConfigByCluster(clusterConfig *ClusterConfig) (*restclient.Config, error) + GetResource(ctx context.Context, namespace string, name string, gvk schema.GroupVersionKind, restConfig *rest.Config) (*ManifestResponse, error) + UpdateResource(ctx context.Context, restConfig *rest.Config, gvk schema.GroupVersionKind, namespace string, k8sRequestPatch string) (*ManifestResponse, error) + DeleteResource(ctx context.Context, restConfig *rest.Config, gvk schema.GroupVersionKind, namespace string, name string, forceDelete bool) (*ManifestResponse, error) + GetPodListByLabel(namespace, label string, clientSet *kubernetes.Clientset) ([]v1.Pod, error) + ExtractK8sServerMajorAndMinorVersion(k8sServerVersion *version.Info) (int, int, error) + GetK8sServerVersion(clientSet *kubernetes.Clientset) (*version.Info, error) + DecodeGroupKindversion(data string) (*schema.GroupVersionKind, error) + GetApiResources(restConfig *rest.Config, includeOnlyVerb string) ([]*K8sApiResource, error) + CreateResources(ctx context.Context, restConfig *rest.Config, manifest string, gvk schema.GroupVersionKind, namespace string) (*ManifestResponse, error) + PatchResourceRequest(ctx context.Context, restConfig *rest.Config, pt types.PatchType, manifest string, name string, namespace string, gvk schema.GroupVersionKind) (*ManifestResponse, error) + GetResourceList(ctx context.Context, restConfig *rest.Config, gvk schema.GroupVersionKind, namespace string) (*ResourceListResponse, bool, error) + GetResourceIfWithAcceptHeader(restConfig *rest.Config, groupVersionKind schema.GroupVersionKind) (resourceIf dynamic.NamespaceableResourceInterface, namespaced bool, err error) + GetPodLogs(ctx context.Context, restConfig *rest.Config, name string, namespace string, sinceTime *metav1.Time, tailLines int, follow bool, containerName string, isPrevContainerLogsEnabled bool) (io.ReadCloser, error) + ListEvents(restConfig *rest.Config, namespace string, groupVersionKind schema.GroupVersionKind, ctx context.Context, name string) (*v1.EventList, error) + GetResourceIf(restConfig *rest.Config, groupVersionKind schema.GroupVersionKind) (resourceIf dynamic.NamespaceableResourceInterface, namespaced bool, err error) + FetchConnectionStatusForCluster(k8sClientSet *kubernetes.Clientset) error + CreateK8sClientSet(restConfig *rest.Config) (*kubernetes.Clientset, error) } func NewK8sUtil(logger *zap.SugaredLogger, runTimeConfig *client.RuntimeConfig) *K8sServiceImpl { diff --git a/vendor/modules.txt b/vendor/modules.txt index 400bb905e9d..1cf47e34781 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -354,9 +354,12 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e +# github.com/devtron-labs/common-lib v0.0.10 ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage +github.com/devtron-labs/common-lib/cloud-provider-identifier +github.com/devtron-labs/common-lib/cloud-provider-identifier/bean +github.com/devtron-labs/common-lib/cloud-provider-identifier/providers github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/pubsub-lib diff --git a/wire_gen.go b/wire_gen.go index 5f0c1bbdc91..6c0ec6a8d54 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -10,6 +10,7 @@ import ( "github.com/devtron-labs/authenticator/apiToken" client2 "github.com/devtron-labs/authenticator/client" "github.com/devtron-labs/authenticator/middleware" + "github.com/devtron-labs/common-lib/cloud-provider-identifier" "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" @@ -726,7 +727,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl) + providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) + telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl) if err != nil { return nil, err }