Endpoint plugins configurable via YAML (#4751)

* Add proxy single request API endpoint

* Add endpoint plugin generation from a YAML file

* Deploy to CF from private GitHub and GitLab repos

* Use subtypes for YAML generated endpoints

* Nicer YAML format; ignore unknown endpoint types in the DB

src/jetstream/http_basic_requests.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"encoding/base64"
+	"errors"
 	"net/http"
 
 	log "github.com/sirupsen/logrus"
@@ -20,3 +22,20 @@ func (p *portalProxy) doHttpBasicFlowRequest(cnsiRequest *interfaces.CNSIRequest
 	return p.DoAuthFlowRequest(cnsiRequest, req, authHandler)
 
 }
+
+func (p *portalProxy) doTokenBearerFlowRequest(cnsiRequest *interfaces.CNSIRequest, req *http.Request) (*http.Response, error) {
+	log.Debug("doTokenBearerFlowRequest")
+
+	authHandler := func(tokenRec interfaces.TokenRecord, cnsi interfaces.CNSIRecord) (*http.Response, error) {
+		authTokenDecodedBytes, err := base64.StdEncoding.DecodeString(tokenRec.AuthToken)
+		if err != nil {
+			return nil, errors.New("Failed to decode auth token")
+		}
+
+		// Token auth has no token refresh or expiry - so much simpler than the OAuth flow
+		req.Header.Set("Authorization", "bearer "+string(authTokenDecodedBytes))
+		client := p.GetHttpClientForRequest(req, cnsi.SkipSSLValidation)
+		return client.Do(req)
+	}
+	return p.DoAuthFlowRequest(cnsiRequest, req, authHandler)
+}

src/jetstream/info.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces"
 	"github.com/labstack/echo/v4"
+	log "github.com/sirupsen/logrus"
 )
 
 // Endpoint - This represents the CNSI endpoint
@@ -97,7 +98,14 @@ func (p *portalProxy) getInfo(c echo.Context) (*interfaces.Info, error) {
 			endpoint.SystemSharedToken = token.SystemShared
 		}
 		cnsiType := cnsi.CNSIType
-		s.Endpoints[cnsiType][cnsi.GUID] = endpoint
+
+		_, ok = s.Endpoints[cnsiType]
+		if ok {
+			s.Endpoints[cnsiType][cnsi.GUID] = endpoint
+		} else {
+			// definitions of YAML-defined plugins may be removed
+			log.Warnf("Unknown endpoint type %q encountered in the DB", cnsiType)
+		}
 	}
 
 	// Allow plugin to modify the info data

src/jetstream/load_plugins.go

@@ -3,13 +3,17 @@ package main
 import (
 	"github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/cloudfoundry-incubator/stratos/src/jetstream/plugins/yamlgenerated"
 )
 
 func (pp *portalProxy) loadPlugins() {
 
 	pp.Plugins = make(map[string]interfaces.StratosPlugin)
 	log.Info("Initialising plugins")
 
+	yamlgenerated.MakePluginsFromConfig()
+
 	for name := range interfaces.PluginInits {
 		addPlugin(pp, name)
 	}

src/jetstream/main.go

@@ -698,6 +698,15 @@ func newPortalProxy(pc interfaces.PortalConfig, dcp *sql.DB, ss HttpSessionStore
 		UserInfo: pp.GetCNSIUserFromBasicToken,
 	})
 
+	// Generic Token Bearer Auth
+	pp.AddAuthProvider(interfaces.AuthTypeBearer, interfaces.AuthProvider{
+		Handler: pp.doTokenBearerFlowRequest,
+		UserInfo: func(cnsiGUID string, cfTokenRecord *interfaces.TokenRecord) (*interfaces.ConnectedUser, bool) {
+			// don't fetch user info for the generic token auth
+			return &interfaces.ConnectedUser{}, false
+		},
+	})
+
 	// OIDC
 	pp.AddAuthProvider(interfaces.AuthTypeOIDC, interfaces.AuthProvider{
 		Handler: pp.DoOidcFlowRequest,
@@ -1025,6 +1034,11 @@ func (p *portalProxy) registerRoutes(e *echo.Echo, needSetupMiddleware bool) {
 	// Proxy single request
 	stableAPIGroup.GET("/proxy/:uuid/*", p.ProxySingleRequest)
 
+	sessionAuthGroup := sessionGroup.Group("/auth")
+
+	// Connect to Endpoint (SSO)
+	sessionAuthGroup.GET("/tokens", p.ssoLoginToCNSI)
+
 	// Info
 	sessionGroup.GET("/info", p.info)
 

src/jetstream/plugins.yaml

@@ -0,0 +1,5 @@
+- name: git.private_github
+  auth_type: HttpBasic
+- name: git.private_gitlab
+  auth_type: Bearer
+

src/jetstream/plugins/cfapppush/deploy.go

@@ -1,10 +1,12 @@
 package cfapppush
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -68,6 +70,11 @@ const (
 	OVERRIDES_SUPPLIED
 )
 
+const (
+	SCM_TYPE_GITHUB = "github"
+	SCM_TYPE_GITLAB = "gitlab"
+)
+
 const (
 	stratosProjectKey = "STRATOS_PROJECT"
 )
@@ -82,6 +89,7 @@ func (cfAppPush *CFAppPush) deploy(echoContext echo.Context) error {
 
 	// App ID is this is a redeploy
 	appID := echoContext.QueryParam("app")
+	userGUID := echoContext.Get("user_id").(string)
 
 	log.Debug("UpgradeToWebSocket")
 	clientWebSocket, pingTicker, err := interfaces.UpgradeToWebSocket(echoContext)
@@ -121,7 +129,7 @@ func (cfAppPush *CFAppPush) deploy(echoContext echo.Context) error {
 	// Get the source, depending on the source type
 	switch msg.Type {
 	case SOURCE_GITSCM:
-		stratosProject, appDir, err = getGitSCMSource(clientWebSocket, tempDir, msg)
+		stratosProject, appDir, err = cfAppPush.getGitSCMSource(clientWebSocket, tempDir, msg, userGUID)
 	case SOURCE_FOLDER:
 		stratosProject, appDir, err = getFolderSource(clientWebSocket, tempDir, msg)
 	case SOURCE_GITURL:
@@ -369,7 +377,7 @@ func getFolderSource(clientWebSocket *websocket.Conn, tempDir string, msg Socket
 	return stratosProject, tempDir, nil
 }
 
-func getGitSCMSource(clientWebSocket *websocket.Conn, tempDir string, msg SocketMessage) (StratosProject, string, error) {
+func (cfAppPush *CFAppPush) getGitSCMSource(clientWebSocket *websocket.Conn, tempDir string, msg SocketMessage, userGUID string) (StratosProject, string, error) {
 	var (
 		err error
 	)
@@ -380,7 +388,57 @@ func getGitSCMSource(clientWebSocket *websocket.Conn, tempDir string, msg Socket
 		return StratosProject{}, tempDir, err
 	}
 
-	log.Debugf("GitSCM SCM: %s, Source: %s, branch %s, url: %s", info.SCM, info.Project, info.Branch, info.URL)
+	loggerURL := info.URL
+
+	if len(info.EndpointGUID) != 0 {
+		tokenRecord, isTokenFound := cfAppPush.portalProxy.GetCNSITokenRecord(info.EndpointGUID, userGUID)
+		if isTokenFound != true {
+			err := fmt.Errorf("No token found for endpoint %s", info.EndpointGUID)
+			log.Errorf("%+v", err)
+			return StratosProject{}, tempDir, err
+		}
+
+		authTokenDecodedBytes, err := base64.StdEncoding.DecodeString(tokenRecord.AuthToken)
+		if err != nil {
+			return StratosProject{}, tempDir, errors.New("Failed to decode auth token")
+		}
+
+		parsedURL, err := url.Parse(info.URL)
+		if err != nil {
+			return StratosProject{}, tempDir, errors.New("Failed to parse SCM URL")
+		}
+
+		var (
+			username string
+			password string
+		)
+
+		switch info.SCM {
+		case SCM_TYPE_GITHUB:
+			// GitHub API uses basic HTTP auth, username and password are stored in the DB
+			pieces := strings.SplitN(string(authTokenDecodedBytes), ":", 2)
+			username, password = pieces[0], pieces[1]
+		case SCM_TYPE_GITLAB:
+			// GitLab API uses bearer token auth, the username is supplied by the frontend
+			username = info.Username
+			password = string(authTokenDecodedBytes)
+		default:
+			return StratosProject{}, tempDir, fmt.Errorf("Unknown SCM type '%s'", info.SCM)
+		}
+
+		if len(username) == 0 {
+			return StratosProject{}, tempDir, errors.New("Username is empty")
+		}
+
+		// mask the credentials for the logs
+		parsedURL.User = url.UserPassword("REDACTED", "REDACTED")
+		loggerURL = parsedURL.String()
+
+		parsedURL.User = url.UserPassword(username, password)
+		info.URL = parsedURL.String()
+	}
+
+	log.Debugf("GitSCM SCM: %s, Source: %s, branch %s, url: %s", info.SCM, info.Project, info.Branch, loggerURL)
 	cloneDetails := CloneDetails{
 		Url:    info.URL,
 		Branch: info.Branch,

src/jetstream/plugins/cfapppush/types.go

@@ -40,11 +40,13 @@ type DeploySource struct {
 // Structure used to provide metadata about the GitHub source
 type GitSCMSourceInfo struct {
 	DeploySource
-	Project    string `json:"project"`
-	Branch     string `json:"branch"`
-	URL        string `json:"url"`
-	CommitHash string `json:"commit"`
-	SCM        string `json:"scm"`
+	Project      string `json:"project"`
+	Branch       string `json:"branch"`
+	URL          string `json:"url"`
+	CommitHash   string `json:"commit"`
+	SCM          string `json:"scm"`
+	EndpointGUID string `json:"endpoint_guid"` // credentials of which to use, e.g. of a private GitHub instance
+	Username     string `json:"username"`      // GitLab username has to be supplied by the frontend
 }
 
 // Structure used to provide metadata about the Git Url source