Merge pull request #1196 from memphisdev/github-integration-function

add github-integration-function

db/db.go

@@ -3880,7 +3880,7 @@ func DeleteIntegration(name string, tenantName string) error {
 	return nil
 }
 
-func InsertNewIntegration(tenantName string, name string, keys map[string]string, properties map[string]bool) (models.Integration, error) {
+func InsertNewIntegration(tenantName string, name string, keys map[string]interface{}, properties map[string]bool) (models.Integration, error) {
 	if tenantName != conf.GlobalAccount {
 		tenantName = strings.ToLower(tenantName)
 	}
@@ -3944,7 +3944,7 @@ func InsertNewIntegration(tenantName string, name string, keys map[string]string
 	return newIntegration, nil
 }
 
-func UpdateIntegration(tenantName string, name string, keys map[string]string, properties map[string]bool) (models.Integration, error) {
+func UpdateIntegration(tenantName string, name string, keys map[string]interface{}, properties map[string]bool) (models.Integration, error) {
 	if tenantName != conf.GlobalAccount {
 		tenantName = strings.ToLower(tenantName)
 	}
@@ -4622,7 +4622,7 @@ func GetAllActiveUsersStations(tenantName string) ([]models.FilteredUser, error)
 	FROM users AS u
 	JOIN stations AS s ON u.id = s.created_by
 	WHERE s.tenant_name=$1 AND username NOT LIKE '$%'` // filter memphis internal users
-	
+
 	stmt, err := conn.Conn().Prepare(ctx, "get_all_active_users_stations", query)
 	if err != nil {
 		return []models.FilteredUser{}, err
@@ -4658,7 +4658,7 @@ func GetAllActiveUsersSchemaVersions(tenantName string) ([]models.FilteredUser,
 	FROM users AS u
 	JOIN schema_versions AS s ON u.id = s.created_by
 	WHERE s.tenant_name=$1 AND username NOT LIKE '$%'` // filter memphis internal users
-	
+
 	stmt, err := conn.Conn().Prepare(ctx, "get_all_active_users_schema_versions", query)
 	if err != nil {
 		return []models.FilteredUser{}, err

go.mod

@@ -45,6 +45,7 @@ require (
 )
 
 require (
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/jackc/puddle/v2 v2.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -86,6 +87,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -113,7 +115,7 @@ require (
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
+	golang.org/x/oauth2 v0.8.0
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect

http_server/routes/integrations.go

@@ -24,6 +24,7 @@ func InitializeIntegrationsRoutes(router *gin.RouterGroup, h *server.Handlers) {
 	integrationsRoutes.POST("/updateIntegration", integrationsHandler.UpdateIntegration)
 	integrationsRoutes.DELETE("/disconnectIntegration", integrationsHandler.DisconnectIntegration)
 	integrationsRoutes.GET("/getIntegrationDetails", integrationsHandler.GetIntegrationDetails)
+	integrationsRoutes.GET("/getSourceCodeBranches", integrationsHandler.GetSourecCodeBranches)
 	integrationsRoutes.GET("/getAllIntegrations", integrationsHandler.GetAllIntegrations)
 	integrationsRoutes.POST("/requestIntegration", integrationsHandler.RequestIntegration) // TODO to be deleted
 }

models/integrations.go

@@ -16,11 +16,11 @@ import (
 )
 
 type Integration struct {
-	ID         int               `json:"id"`
-	Name       string            `json:"name"`
-	Keys       map[string]string `json:"keys"`
-	Properties map[string]bool   `json:"properties"`
-	TenantName string            `json:"tenant_name"`
+	ID         int                    `json:"id"`
+	Name       string                 `json:"name"`
+	Keys       map[string]interface{} `json:"keys"`
+	Properties map[string]bool        `json:"properties"`
+	TenantName string                 `json:"tenant_name"`
 }
 
 type SlackIntegration struct {
@@ -31,18 +31,18 @@ type SlackIntegration struct {
 }
 
 type CreateIntegrationSchema struct {
-	Name       string            `json:"name"`
-	Keys       map[string]string `json:"keys"`
-	Properties map[string]bool   `json:"properties"`
-	UIUrl      string            `json:"ui_url"`
+	Name       string                 `json:"name"`
+	Keys       map[string]interface{} `json:"keys"`
+	Properties map[string]bool        `json:"properties"`
+	UIUrl      string                 `json:"ui_url"`
 }
 
 type CreateIntegration struct {
-	Name       string            `json:"name"`
-	Keys       map[string]string `json:"keys"`
-	Properties map[string]bool   `json:"properties"`
-	UIUrl      string            `json:"ui_url"`
-	TenantName string            `json:"tenant_name"`
+	Name       string                 `json:"name"`
+	Keys       map[string]interface{} `json:"keys"`
+	Properties map[string]bool        `json:"properties"`
+	UIUrl      string                 `json:"ui_url"`
+	TenantName string                 `json:"tenant_name"`
 }
 
 type GetIntegrationDetailsSchema struct {

server/background_tasks.go

@@ -117,6 +117,8 @@ func (s *Server) ListenForIntegrationsUpdateEvents() error {
 				CacheDetails("slack", integrationUpdate.Keys, integrationUpdate.Properties, integrationUpdate.TenantName)
 			case "s3":
 				CacheDetails("s3", integrationUpdate.Keys, integrationUpdate.Properties, integrationUpdate.TenantName)
+			case "github":
+				CacheDetails("github", integrationUpdate.Keys, integrationUpdate.Properties, integrationUpdate.TenantName)
 			default:
 				s.Warnf("[tenant: %v] ListenForIntegrationsUpdateEvents: %s %s", integrationUpdate.TenantName, strings.ToLower(integrationUpdate.Name), "unknown integration")
 				return

server/integrations.go

@@ -13,12 +13,14 @@ package server
 
 import (
 	"errors"
+	"fmt"
 	"memphis/db"
 )
 
 var IntegrationsConcurrentCache *concurrentMap[map[string]interface{}]
 var NotificationFunctionsMap map[string]interface{}
 var StorageFunctionsMap map[string]interface{}
+var SourceCodeManagementFunctionsMap map[string]map[string]interface{}
 
 const PoisonMAlert = "poison_message_alert"
 const SchemaVAlert = "schema_validation_fail_alert"
@@ -28,8 +30,12 @@ func InitializeIntegrations() error {
 	IntegrationsConcurrentCache = NewConcurrentMap[map[string]interface{}]()
 	NotificationFunctionsMap = make(map[string]interface{})
 	StorageFunctionsMap = make(map[string]interface{})
+	SourceCodeManagementFunctionsMap = make(map[string]map[string]interface{})
 	NotificationFunctionsMap["slack"] = sendMessageToSlackChannel
 	StorageFunctionsMap["s3"] = serv.uploadToS3Storage
+	SourceCodeManagementFunctionsMap["github"] = make(map[string]interface{})
+	SourceCodeManagementFunctionsMap["github"]["get_all_repos"] = serv.getGithubRepositories
+	SourceCodeManagementFunctionsMap["github"]["get_all_branches"] = serv.getGithubBranches
 
 	err := InitializeConnections()
 	if err != nil {
@@ -49,13 +55,13 @@ func InitializeConnections() error {
 	}
 	for _, integration := range integrations {
 		if value, ok := integration.Keys["secret_key"]; ok {
-			decryptedValue, err := DecryptAES(key, value)
+			decryptedValue, err := DecryptAES(key, value.(string))
 			if err != nil {
 				return err
 			}
 			integration.Keys["secret_key"] = decryptedValue
 		} else if value, ok := integration.Keys["auth_token"]; ok {
-			decryptedValue, err := DecryptAES(key, value)
+			decryptedValue, err := DecryptAES(key, value.(string))
 			if err != nil {
 				return err
 			}
@@ -66,13 +72,14 @@ func InitializeConnections() error {
 	return nil
 }
 
-func CacheDetails(integrationType string, keys map[string]string, properties map[string]bool, tenantName string) {
+func CacheDetails(integrationType string, keys map[string]interface{}, properties map[string]bool, tenantName string) {
 	switch integrationType {
 	case "slack":
 		cacheDetailsSlack(keys, properties, tenantName)
 	case "s3":
 		cacheDetailsS3(keys, properties, tenantName)
-
+	case "github":
+		cacheDetailsGithub(keys, properties, tenantName)
 	}
 }
 
@@ -109,18 +116,18 @@ func encryptUnencryptedKeysByIntegrationType(integrationType, keyTitle string, t
 	needToEncrypt := false
 	key := getAESKey()
 	if value, ok := integration.Keys["secret_key"]; ok {
-		_, err := DecryptAES(key, value)
+		_, err := DecryptAES(key, value.(string))
 		if err != nil {
 			needToEncrypt = true
 		}
 	} else if value, ok := integration.Keys["auth_token"]; ok {
-		_, err := DecryptAES(key, value)
+		_, err := DecryptAES(key, value.(string))
 		if err != nil {
 			needToEncrypt = true
 		}
 	}
 	if needToEncrypt {
-		encryptedValue, err := EncryptAES([]byte(integration.Keys[keyTitle]))
+		encryptedValue, err := EncryptAES([]byte(integration.Keys[keyTitle].(string)))
 		if err != nil {
 			return err
 		}
@@ -176,3 +183,31 @@ func deleteIntegrationFromTenant(tenantName string, integrationType string, inte
 		integrations.m[tenantName] = i
 	}
 }
+
+func hideIntegrationSecretKey(secretKey string) string {
+	if secretKey != "" {
+		lastCharsSecretKey := secretKey[len(secretKey)-4:]
+		secretKey = "****" + lastCharsSecretKey
+		return secretKey
+	}
+	return secretKey
+}
+
+func copyStringMapToInterfaceMap(srcMap map[string]string) map[string]interface{} {
+	destMap := make(map[string]interface{})
+
+	for k, v := range srcMap {
+		destMap[k] = v
+	}
+	return destMap
+}
+
+func GetKeysAsStringMap(keys map[string]interface{}) map[string]string {
+	stringMap := make(map[string]string)
+
+	for k, v := range keys {
+		stringValue := fmt.Sprintf("%v", v)
+		stringMap[k] = stringValue
+	}
+	return stringMap
+}