From bb3a4f4c7360d80aefd068aafe5d0f87c37054ef Mon Sep 17 00:00:00 2001 From: Ivan Kapelyukhin Date: Thu, 19 Nov 2020 12:58:53 +0100 Subject: [PATCH] Endpoint plugins configurable via YAML (#4751) * Add proxy single request API endpoint * Add endpoint plugin generation from a YAML file * Deploy to CF from private GitHub and GitLab repos * Use subtypes for YAML generated endpoints * Nicer YAML format; ignore unknown endpoint types in the DB --- src/jetstream/http_basic_requests.go | 19 ++ src/jetstream/info.go | 10 +- src/jetstream/load_plugins.go | 4 + src/jetstream/main.go | 14 ++ src/jetstream/plugins.yaml | 5 + src/jetstream/plugins/cfapppush/deploy.go | 64 +++++- src/jetstream/plugins/cfapppush/types.go | 12 +- src/jetstream/plugins/yamlgenerated/main.go | 200 ++++++++++++++++++ .../repository/interfaces/structs.go | 4 + 9 files changed, 323 insertions(+), 9 deletions(-) create mode 100644 src/jetstream/plugins.yaml create mode 100644 src/jetstream/plugins/yamlgenerated/main.go diff --git a/src/jetstream/http_basic_requests.go b/src/jetstream/http_basic_requests.go index 3e11f57230..a039beef7b 100644 --- a/src/jetstream/http_basic_requests.go +++ b/src/jetstream/http_basic_requests.go @@ -1,6 +1,8 @@ package main import ( + "encoding/base64" + "errors" "net/http" log "github.com/sirupsen/logrus" @@ -20,3 +22,20 @@ func (p *portalProxy) doHttpBasicFlowRequest(cnsiRequest *interfaces.CNSIRequest return p.DoAuthFlowRequest(cnsiRequest, req, authHandler) } + +func (p *portalProxy) doTokenBearerFlowRequest(cnsiRequest *interfaces.CNSIRequest, req *http.Request) (*http.Response, error) { + log.Debug("doTokenBearerFlowRequest") + + authHandler := func(tokenRec interfaces.TokenRecord, cnsi interfaces.CNSIRecord) (*http.Response, error) { + authTokenDecodedBytes, err := base64.StdEncoding.DecodeString(tokenRec.AuthToken) + if err != nil { + return nil, errors.New("Failed to decode auth token") + } + + // Token auth has no token refresh or expiry - so much simpler than the OAuth flow + req.Header.Set("Authorization", "bearer "+string(authTokenDecodedBytes)) + client := p.GetHttpClientForRequest(req, cnsi.SkipSSLValidation) + return client.Do(req) + } + return p.DoAuthFlowRequest(cnsiRequest, req, authHandler) +} diff --git a/src/jetstream/info.go b/src/jetstream/info.go index 13c1097e94..96656ae83b 100644 --- a/src/jetstream/info.go +++ b/src/jetstream/info.go @@ -8,6 +8,7 @@ import ( "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" "github.com/labstack/echo/v4" + log "github.com/sirupsen/logrus" ) // Endpoint - This represents the CNSI endpoint @@ -97,7 +98,14 @@ func (p *portalProxy) getInfo(c echo.Context) (*interfaces.Info, error) { endpoint.SystemSharedToken = token.SystemShared } cnsiType := cnsi.CNSIType - s.Endpoints[cnsiType][cnsi.GUID] = endpoint + + _, ok = s.Endpoints[cnsiType] + if ok { + s.Endpoints[cnsiType][cnsi.GUID] = endpoint + } else { + // definitions of YAML-defined plugins may be removed + log.Warnf("Unknown endpoint type %q encountered in the DB", cnsiType) + } } // Allow plugin to modify the info data diff --git a/src/jetstream/load_plugins.go b/src/jetstream/load_plugins.go index 406b729ab4..173db09b96 100644 --- a/src/jetstream/load_plugins.go +++ b/src/jetstream/load_plugins.go @@ -3,6 +3,8 @@ package main import ( "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" log "github.com/sirupsen/logrus" + + "github.com/cloudfoundry-incubator/stratos/src/jetstream/plugins/yamlgenerated" ) func (pp *portalProxy) loadPlugins() { @@ -10,6 +12,8 @@ func (pp *portalProxy) loadPlugins() { pp.Plugins = make(map[string]interfaces.StratosPlugin) log.Info("Initialising plugins") + yamlgenerated.MakePluginsFromConfig() + for name := range interfaces.PluginInits { addPlugin(pp, name) } diff --git a/src/jetstream/main.go b/src/jetstream/main.go index 0014731cc9..d391e8077a 100644 --- a/src/jetstream/main.go +++ b/src/jetstream/main.go @@ -698,6 +698,15 @@ func newPortalProxy(pc interfaces.PortalConfig, dcp *sql.DB, ss HttpSessionStore UserInfo: pp.GetCNSIUserFromBasicToken, }) + // Generic Token Bearer Auth + pp.AddAuthProvider(interfaces.AuthTypeBearer, interfaces.AuthProvider{ + Handler: pp.doTokenBearerFlowRequest, + UserInfo: func(cnsiGUID string, cfTokenRecord *interfaces.TokenRecord) (*interfaces.ConnectedUser, bool) { + // don't fetch user info for the generic token auth + return &interfaces.ConnectedUser{}, false + }, + }) + // OIDC pp.AddAuthProvider(interfaces.AuthTypeOIDC, interfaces.AuthProvider{ Handler: pp.DoOidcFlowRequest, @@ -1025,6 +1034,11 @@ func (p *portalProxy) registerRoutes(e *echo.Echo, needSetupMiddleware bool) { // Proxy single request stableAPIGroup.GET("/proxy/:uuid/*", p.ProxySingleRequest) + sessionAuthGroup := sessionGroup.Group("/auth") + + // Connect to Endpoint (SSO) + sessionAuthGroup.GET("/tokens", p.ssoLoginToCNSI) + // Info sessionGroup.GET("/info", p.info) diff --git a/src/jetstream/plugins.yaml b/src/jetstream/plugins.yaml new file mode 100644 index 0000000000..ecffc42e3e --- /dev/null +++ b/src/jetstream/plugins.yaml @@ -0,0 +1,5 @@ +- name: git.private_github + auth_type: HttpBasic +- name: git.private_gitlab + auth_type: Bearer + diff --git a/src/jetstream/plugins/cfapppush/deploy.go b/src/jetstream/plugins/cfapppush/deploy.go index ffa7b00813..4ea0c2344e 100644 --- a/src/jetstream/plugins/cfapppush/deploy.go +++ b/src/jetstream/plugins/cfapppush/deploy.go @@ -1,10 +1,12 @@ package cfapppush import ( + "encoding/base64" "encoding/json" "errors" "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "strings" @@ -68,6 +70,11 @@ const ( OVERRIDES_SUPPLIED ) +const ( + SCM_TYPE_GITHUB = "github" + SCM_TYPE_GITLAB = "gitlab" +) + const ( stratosProjectKey = "STRATOS_PROJECT" ) @@ -82,6 +89,7 @@ func (cfAppPush *CFAppPush) deploy(echoContext echo.Context) error { // App ID is this is a redeploy appID := echoContext.QueryParam("app") + userGUID := echoContext.Get("user_id").(string) log.Debug("UpgradeToWebSocket") clientWebSocket, pingTicker, err := interfaces.UpgradeToWebSocket(echoContext) @@ -121,7 +129,7 @@ func (cfAppPush *CFAppPush) deploy(echoContext echo.Context) error { // Get the source, depending on the source type switch msg.Type { case SOURCE_GITSCM: - stratosProject, appDir, err = getGitSCMSource(clientWebSocket, tempDir, msg) + stratosProject, appDir, err = cfAppPush.getGitSCMSource(clientWebSocket, tempDir, msg, userGUID) case SOURCE_FOLDER: stratosProject, appDir, err = getFolderSource(clientWebSocket, tempDir, msg) case SOURCE_GITURL: @@ -369,7 +377,7 @@ func getFolderSource(clientWebSocket *websocket.Conn, tempDir string, msg Socket return stratosProject, tempDir, nil } -func getGitSCMSource(clientWebSocket *websocket.Conn, tempDir string, msg SocketMessage) (StratosProject, string, error) { +func (cfAppPush *CFAppPush) getGitSCMSource(clientWebSocket *websocket.Conn, tempDir string, msg SocketMessage, userGUID string) (StratosProject, string, error) { var ( err error ) @@ -380,7 +388,57 @@ func getGitSCMSource(clientWebSocket *websocket.Conn, tempDir string, msg Socket return StratosProject{}, tempDir, err } - log.Debugf("GitSCM SCM: %s, Source: %s, branch %s, url: %s", info.SCM, info.Project, info.Branch, info.URL) + loggerURL := info.URL + + if len(info.EndpointGUID) != 0 { + tokenRecord, isTokenFound := cfAppPush.portalProxy.GetCNSITokenRecord(info.EndpointGUID, userGUID) + if isTokenFound != true { + err := fmt.Errorf("No token found for endpoint %s", info.EndpointGUID) + log.Errorf("%+v", err) + return StratosProject{}, tempDir, err + } + + authTokenDecodedBytes, err := base64.StdEncoding.DecodeString(tokenRecord.AuthToken) + if err != nil { + return StratosProject{}, tempDir, errors.New("Failed to decode auth token") + } + + parsedURL, err := url.Parse(info.URL) + if err != nil { + return StratosProject{}, tempDir, errors.New("Failed to parse SCM URL") + } + + var ( + username string + password string + ) + + switch info.SCM { + case SCM_TYPE_GITHUB: + // GitHub API uses basic HTTP auth, username and password are stored in the DB + pieces := strings.SplitN(string(authTokenDecodedBytes), ":", 2) + username, password = pieces[0], pieces[1] + case SCM_TYPE_GITLAB: + // GitLab API uses bearer token auth, the username is supplied by the frontend + username = info.Username + password = string(authTokenDecodedBytes) + default: + return StratosProject{}, tempDir, fmt.Errorf("Unknown SCM type '%s'", info.SCM) + } + + if len(username) == 0 { + return StratosProject{}, tempDir, errors.New("Username is empty") + } + + // mask the credentials for the logs + parsedURL.User = url.UserPassword("REDACTED", "REDACTED") + loggerURL = parsedURL.String() + + parsedURL.User = url.UserPassword(username, password) + info.URL = parsedURL.String() + } + + log.Debugf("GitSCM SCM: %s, Source: %s, branch %s, url: %s", info.SCM, info.Project, info.Branch, loggerURL) cloneDetails := CloneDetails{ Url: info.URL, Branch: info.Branch, diff --git a/src/jetstream/plugins/cfapppush/types.go b/src/jetstream/plugins/cfapppush/types.go index 650098c5e8..4fdd26db11 100644 --- a/src/jetstream/plugins/cfapppush/types.go +++ b/src/jetstream/plugins/cfapppush/types.go @@ -40,11 +40,13 @@ type DeploySource struct { // Structure used to provide metadata about the GitHub source type GitSCMSourceInfo struct { DeploySource - Project string `json:"project"` - Branch string `json:"branch"` - URL string `json:"url"` - CommitHash string `json:"commit"` - SCM string `json:"scm"` + Project string `json:"project"` + Branch string `json:"branch"` + URL string `json:"url"` + CommitHash string `json:"commit"` + SCM string `json:"scm"` + EndpointGUID string `json:"endpoint_guid"` // credentials of which to use, e.g. of a private GitHub instance + Username string `json:"username"` // GitLab username has to be supplied by the frontend } // Structure used to provide metadata about the Git Url source diff --git a/src/jetstream/plugins/yamlgenerated/main.go b/src/jetstream/plugins/yamlgenerated/main.go new file mode 100644 index 0000000000..200283f506 --- /dev/null +++ b/src/jetstream/plugins/yamlgenerated/main.go @@ -0,0 +1,200 @@ +package yamlgenerated + +import ( + "encoding/base64" + "errors" + "fmt" + "io/ioutil" + "net/url" + "strings" + + "github.com/cloudfoundry-incubator/stratos/src/jetstream/repository/interfaces" + "github.com/labstack/echo/v4" + "gopkg.in/yaml.v2" + + log "github.com/sirupsen/logrus" +) + +type GeneratedPlugin struct { + initMethod func() error + middlewarePlugin func() (interfaces.MiddlewarePlugin, error) + endpointPlugin func() (interfaces.EndpointPlugin, error) + routePlugin func() (interfaces.RoutePlugin, error) +} + +var authTypeToConnectTypeMap = map[string]string{ + interfaces.AuthTypeHttpBasic: interfaces.AuthConnectTypeCreds, + interfaces.AuthTypeBearer: interfaces.AuthConnectTypeBearer, +} + +func (gp GeneratedPlugin) Init() error { return gp.initMethod() } +func (gp GeneratedPlugin) GetMiddlewarePlugin() (interfaces.MiddlewarePlugin, error) { + return gp.middlewarePlugin() +} +func (gp GeneratedPlugin) GetEndpointPlugin() (interfaces.EndpointPlugin, error) { + return gp.endpointPlugin() +} +func (gp GeneratedPlugin) GetRoutePlugin() (interfaces.RoutePlugin, error) { + return gp.routePlugin() +} + +type GeneratedEndpointPlugin struct { + portalProxy interfaces.PortalProxy + endpointType string + authTypes map[string]string +} + +func (gep GeneratedEndpointPlugin) GetType() string { + return gep.endpointType +} + +func (gep GeneratedEndpointPlugin) Register(ec echo.Context) error { + return gep.portalProxy.RegisterEndpoint(ec, gep.Info) +} + +func (gep GeneratedEndpointPlugin) Validate(userGUID string, cnsiRecord interfaces.CNSIRecord, tokenRecord interfaces.TokenRecord) error { + return nil +} + +func (gep GeneratedEndpointPlugin) Connect(ec echo.Context, cnsiRecord interfaces.CNSIRecord, userId string) (*interfaces.TokenRecord, bool, error) { + params := new(interfaces.LoginToCNSIParams) + err := interfaces.BindOnce(params, ec) + if err != nil { + return nil, false, err + } + + authType, ok := gep.authTypes[cnsiRecord.SubType] + if !ok { + return nil, false, fmt.Errorf("Unknown subtype %q for endpoint type %q", cnsiRecord.SubType, gep.GetType()) + } + + expectedConnectType, ok := authTypeToConnectTypeMap[authType] + if !ok { + return nil, false, fmt.Errorf("Unknown authentication type %q for plugin %q", authType, gep.GetType()) + } + + if expectedConnectType != params.ConnectType { + return nil, false, fmt.Errorf("Only %q connect type is supported for %q.%q endpoints", expectedConnectType, gep.GetType(), cnsiRecord.SubType) + } + + var tr *interfaces.TokenRecord + + switch params.ConnectType { + case interfaces.AuthConnectTypeCreds: + if len(params.Username) == 0 || len(params.Password) == 0 { + return nil, false, errors.New("Need username and password") + } + + authString := fmt.Sprintf("%s:%s", params.Username, params.Password) + base64EncodedAuthString := base64.StdEncoding.EncodeToString([]byte(authString)) + + tr = &interfaces.TokenRecord{ + AuthType: interfaces.AuthTypeHttpBasic, + AuthToken: base64EncodedAuthString, + RefreshToken: params.Username, + } + case interfaces.AuthConnectTypeBearer: + authString := ec.FormValue("token") + base64EncodedAuthString := base64.StdEncoding.EncodeToString([]byte(authString)) + + tr = &interfaces.TokenRecord{ + AuthType: interfaces.AuthTypeBearer, + AuthToken: base64EncodedAuthString, + RefreshToken: "token", // DB needs a non-empty value + } + } + + return tr, false, nil +} + +func (gep GeneratedEndpointPlugin) Info(apiEndpoint string, skipSSLValidation bool) (interfaces.CNSIRecord, interface{}, error) { + var dummy interface{} + var newCNSI interfaces.CNSIRecord + + newCNSI.CNSIType = gep.GetType() + + _, err := url.Parse(apiEndpoint) + if err != nil { + return newCNSI, nil, err + } + + newCNSI.TokenEndpoint = apiEndpoint + newCNSI.AuthorizationEndpoint = apiEndpoint + + return newCNSI, dummy, nil +} + +func (gep GeneratedEndpointPlugin) UpdateMetadata(info *interfaces.Info, userGUID string, echoContext echo.Context) { + // no-op +} + +type PluginConfig struct { + Name string `yaml:"name"` + AuthType string `yaml:"auth_type"` +} + +func MakePluginsFromConfig() { + log.Debug("MakePluginsFromConfig") + + var config []PluginConfig + + yamlFile, err := ioutil.ReadFile("plugins.yaml") + if err != nil { + log.Errorf("Can't generate plugins from YAML: %v ", err) + return + } + + err = yaml.Unmarshal(yamlFile, &config) + if err != nil { + log.Errorf("Failed to unmarshal YAML: %v ", err) + return + } + + plugins := make(map[string]map[string]string) + + for _, plugin := range config { + if len(plugin.Name) == 0 { + log.Errorf("Plugin must have a name") + return + } + + log.Debugf("Generating plugin %s", plugin.Name) + + pieces := strings.SplitN(plugin.Name, ".", 2) + endpointType, endpointSubtype := pieces[0], "" + + if len(pieces) > 1 { + endpointSubtype = pieces[1] + } + + _, ok := plugins[endpointType] + if !ok { + plugins[endpointType] = make(map[string]string) + } + + plugins[endpointType][endpointSubtype] = plugin.AuthType + } + + for endpointType, authTypes := range plugins { + gep := GeneratedEndpointPlugin{} + gep.endpointType = endpointType + gep.authTypes = authTypes + + gp := GeneratedPlugin{} + gp.initMethod = func() error { return nil } + gp.endpointPlugin = func() (interfaces.EndpointPlugin, error) { return gep, nil } + gp.middlewarePlugin = func() (interfaces.MiddlewarePlugin, error) { return nil, errors.New("Not implemented") } + gp.routePlugin = func() (interfaces.RoutePlugin, error) { return nil, errors.New("Not implemented") } + + interfaces.AddPlugin( + endpointType, + []string{}, + func(portalProxy interfaces.PortalProxy) (interfaces.StratosPlugin, error) { + log.Debugf("%s -- initializing", endpointType) + + gep.portalProxy = portalProxy + return gp, nil + }, + ) + } +} diff --git a/src/jetstream/repository/interfaces/structs.go b/src/jetstream/repository/interfaces/structs.go index 82b2bf0f31..1fcc7d3c22 100644 --- a/src/jetstream/repository/interfaces/structs.go +++ b/src/jetstream/repository/interfaces/structs.go @@ -79,11 +79,15 @@ const ( AuthTypeOIDC = "OIDC" // AuthTypeHttpBasic means HTTP Basic auth AuthTypeHttpBasic = "HttpBasic" + // AuthTypeBearer is authentication with an API token + AuthTypeBearer = "Bearer" ) const ( // AuthConnectTypeCreds means authenticate with username/password credentials AuthConnectTypeCreds = "creds" + // AuthConnectTypeBearer is authentication with an API token + AuthConnectTypeBearer = "bearer" // AuthConnectTypeNone means no authentication AuthConnectTypeNone = "none" )