adding support for public and private git providers

README.md

@@ -13,6 +13,32 @@ The Devfile Parser library is a Golang module that:
 3. generates Kubernetes objects for the various devfile resources.
 4. defines util functions for the devfile.
 
+## Private Repository Support
+
+Tokens are required to be set in the following cases:
+1. parsing a devfile from a private repository
+2. parsing a devfile containing a parent devfile from a private repository [1]
+3. parsing a devfile from a private repository containing a parent devfile from a public repository [2]
+
+Set the token for the repository:
+```go
+parser.ParserArgs{
+	...
+	URL: <url-to-devfile-on-supported-git-provider>
+	Token: <repo-personal-access-token>
+	...
+}
+```
+Note: The url must also be set with a supported git provider repo url.
+
+For more information about personal access tokens:
+1. [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
+2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token)
+3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/)
+
+[1] Currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository.
+[2] In this scenario, the token will be used to authenticate the main devfile.
+
 ## Usage
 
 The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/github.com/devfile/library). 
@@ -35,7 +61,6 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    devfile, variableWarning, err := devfilePkg.ParseDevfileAndValidate(parserArgs)
    ```
 
-
 2. To override the HTTP request and response timeouts for a devfile with a parent reference from a registry URL, specify the HTTPTimeout value in the parser arguments
    ```go
       // specify the timeout in seconds  
@@ -45,7 +70,6 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
 	  }
    ```
 
-
 3. To get specific content from devfile
    ```go
    // To get all the components from the devfile
@@ -77,7 +101,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
 		},
    })
    ```
-   
+
 4. To get the Kubernetes objects from the devfile, visit [generators.go source file](pkg/devfile/generator/generators.go)
    ```go
     // To get a slice of Kubernetes containers of type corev1.Container from the devfile component containers
@@ -94,7 +118,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
 	}
 	deployment := generator.GetDeployment(deployParams)
    ```
-   
+
 5. To update devfile content
    ```go
    // To update an existing component in devfile object
@@ -131,20 +155,19 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    ```go
    // If the devfile object has been created with devfile path already set, can simply call WriteYamlDevfile to writes the devfile
    err := devfile.WriteYamlDevfile()
-
-
+
    // To write to a devfile from scratch
    // create a new DevfileData with a specific devfile version
    devfileData, err := data.NewDevfileData(devfileVersion)
 
    // set schema version
    devfileData.SetSchemaVersion(devfileVersion)
-   
+
    // add devfile content use library APIs
    devfileData.AddComponents([]v1.Component{...})
    devfileData.AddCommands([]v1.Commands{...})
    ......
-   
+
    // create a new DevfileCtx
    ctx := devfileCtx.NewDevfileCtx(devfilePath)
    err = ctx.SetAbsPath()
@@ -154,10 +177,11 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
 		Ctx:  ctx,
 		Data: devfileData,
    }
-    
+
    // write to the devfile on disk
    err = devfile.WriteYamlDevfile()
    ```
+
 7. To parse the outerloop Kubernetes/OpenShift component's uri or inline content, call the read and parse functions
    ```go
    // Read the YAML content
@@ -166,6 +190,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    // Get the Kubernetes resources
    resources, err := ParseKubernetesYaml(values)
    ```
+
 8. By default, the parser will set all unset boolean properties to their spec defined default values.  Clients can override this behaviour by specifiying the parser argument `SetBooleanDefaults` to false
    ```go
    setDefaults := false

pkg/devfile/parser/context/content.go

@@ -1,5 +1,5 @@
 //
-// Copyright 2022 Red Hat, Inc.
+// Copyright 2022-2023 Red Hat, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -69,6 +69,9 @@ func (d *DevfileCtx) SetDevfileContent() error {
 	if d.url != "" {
 		// set the client identifier for telemetry
 		params := util.HTTPRequestParams{URL: d.url, TelemetryClientName: util.TelemetryClientName}
+		if d.token != "" {
+			params.Token = d.token
+		}
 		data, err = util.DownloadInMemory(params)
 		if err != nil {
 			return errors.Wrap(err, "error getting devfile info from url")

pkg/devfile/parser/context/context.go

@@ -1,5 +1,5 @@
 //
-// Copyright 2022 Red Hat, Inc.
+// Copyright 2022-2023 Red Hat, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -45,13 +45,16 @@ type DevfileCtx struct {
 	// devfile json schema
 	jsonSchema string
 
-	//url path of the devfile
+	// url path of the devfile
 	url string
 
+	// token is a personal access token used with a private git repo URL
+	token string
+
 	// filesystem for devfile
 	fs filesystem.Filesystem
 
-	// devfile kubernetes components has been coverted from uri to inlined in memory
+	// devfile kubernetes components has been converted from uri to inlined in memory
 	convertUriToInlined bool
 }
 
@@ -70,6 +73,14 @@ func NewURLDevfileCtx(url string) DevfileCtx {
 	}
 }
 
+// NewPrivateURLDevfileCtx returns a new DevfileCtx type object
+func NewPrivateURLDevfileCtx(url string, token string) DevfileCtx {
+	return DevfileCtx{
+		url:   url,
+		token: token,
+	}
+}
+
 // NewByteContentDevfileCtx set devfile content from byte data and returns a new DevfileCtx type object and error
 func NewByteContentDevfileCtx(data []byte) (d DevfileCtx, err error) {
 	err = d.SetDevfileContentFromBytes(data)
@@ -150,6 +161,11 @@ func (d *DevfileCtx) GetURL() string {
 	return d.url
 }
 
+// GetToken func returns current devfile token
+func (d *DevfileCtx) GetToken() string {
+	return d.token
+}
+
 // SetAbsPath sets absolute file path for devfile
 func (d *DevfileCtx) SetAbsPath() (err error) {
 	// Set devfile absolute path

pkg/devfile/parser/context/context_test.go

@@ -83,6 +83,24 @@ func TestPopulateFromInvalidURL(t *testing.T) {
 	})
 }
 
+func TestNewURLDevfileCtx(t *testing.T) {
+	var (
+		token = "fake-token"
+		url   = "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml"
+	)
+
+	{
+		d := NewPrivateURLDevfileCtx(url, token)
+		assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL())
+		assert.Equal(t, "fake-token", d.GetToken())
+	}
+	{
+		d := NewURLDevfileCtx(url)
+		assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL())
+		assert.Equal(t, "", d.GetToken())
+	}
+}
+
 func invalidJsonRawContent200() []byte {
 	return []byte(InvalidDevfileContent)
 }

pkg/devfile/parser/parse.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/devfile/library/v2/pkg/git"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -46,6 +47,42 @@ import (
 	"github.com/pkg/errors"
 )
 
+// downloadGitRepoResources is exposed as a global variable for the purpose of running mock tests
+var downloadGitRepoResources = func(url string, destDir string, httpTimeout *int, token string) error {
+	gitUrl, err := git.NewGitUrlWithURL(url)
+	if err != nil {
+		return err
+	}
+
+	if gitUrl.IsGitProviderRepo() && gitUrl.IsFile {
+		stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources"))
+		if err != nil {
+			return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err)
+		}
+		defer os.RemoveAll(stackDir)
+
+		if !gitUrl.IsPublic(httpTimeout) {
+			err = gitUrl.SetToken(token, httpTimeout)
+			if err != nil {
+				return err
+			}
+		}
+
+		err = gitUrl.CloneGitRepo(stackDir)
+		if err != nil {
+			return err
+		}
+
+		dir := path.Dir(path.Join(stackDir, gitUrl.Path))
+		err = git.CopyAllDirFiles(dir, destDir)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // ParseDevfile func validates the devfile integrity.
 // Creates devfile context and runtime objects
 func parseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) {
@@ -97,6 +134,8 @@ type ParserArgs struct {
 	// RegistryURLs is a list of registry hosts which parser should pull parent devfile from.
 	// If registryUrl is defined in devfile, this list will be ignored.
 	RegistryURLs []string
+	// Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo URL
+	Token string
 	// DefaultNamespace is the default namespace to use
 	// If namespace is defined under devfile's parent kubernetes object, this namespace will be ignored.
 	DefaultNamespace string
@@ -124,7 +163,11 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
 	} else if args.Path != "" {
 		d.Ctx = devfileCtx.NewDevfileCtx(args.Path)
 	} else if args.URL != "" {
-		d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
+		if args.Token != "" {
+			d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(args.URL, args.Token)
+		} else {
+			d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
+		}
 	} else {
 		return d, errors.Wrap(err, "the devfile source is not provided")
 	}
@@ -431,17 +474,17 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 			return DevfileObj{}, fmt.Errorf("failed to resolve parent uri, devfile context is missing absolute url and path to devfile. %s", resolveImportReference(importReference))
 		}
 
-		d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
-		if strings.Contains(newUri, "raw.githubusercontent.com") {
-			urlComponents, err := util.GetGitUrlComponentsFromRaw(newUri)
-			if err != nil {
-				return DevfileObj{}, err
-			}
-			destDir := path.Dir(curDevfileCtx.GetAbsPath())
-			err = getResourcesFromGit(urlComponents, destDir)
-			if err != nil {
-				return DevfileObj{}, err
-			}
+		token := curDevfileCtx.GetToken()
+		if token != "" {
+			d.Ctx = devfileCtx.NewPrivateURLDevfileCtx(newUri, token)
+		} else {
+			d.Ctx = devfileCtx.NewURLDevfileCtx(newUri)
+		}
+
+		destDir := path.Dir(curDevfileCtx.GetAbsPath())
+		err = downloadGitRepoResources(newUri, destDir, tool.httpTimeout, token)
+		if err != nil {
+			return DevfileObj{}, err
 		}
 	}
 	importReference.Uri = newUri
@@ -450,27 +493,6 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
 	return populateAndParseDevfile(d, newResolveCtx, tool, true)
 }
 
-func getResourcesFromGit(gitUrlComponents map[string]string, destDir string) error {
-	stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources"))
-	if err != nil {
-		return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err)
-	}
-	defer os.RemoveAll(stackDir)
-
-	err = util.CloneGitRepo(gitUrlComponents, stackDir)
-	if err != nil {
-		return err
-	}
-
-	dir := path.Dir(path.Join(stackDir, gitUrlComponents["file"]))
-	err = util.CopyAllDirFiles(dir, destDir)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutionContextTree, tool resolverTools) (d DevfileObj, err error) {
 	id := importReference.Id
 	registryURL := importReference.RegistryUrl
@@ -839,6 +861,9 @@ func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx) ([]byte
 			newUri = uri
 		}
 		params := util.HTTPRequestParams{URL: newUri}
+		if d.GetToken() != "" {
+			params.Token = d.GetToken()
+		}
 		data, err = util.DownloadInMemory(params)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error getting kubernetes resources definition information")