feat: Add Support for Remote URLs in Module Config

cmd/kyma/alpha/create/module/module.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"maps"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -253,6 +254,7 @@ func (cmd *command) Run(ctx context.Context) error {
 	}
 
 	modDef, modCnf, err := cmd.moduleDefinitionFromOptions()
+	defer module.DeleteTempFiles()
 
 	if err != nil {
 		return err
@@ -567,13 +569,28 @@ func (cmd *command) moduleDefinitionFromOptions() (*module.Definition, *Config,
 
 	var defaultCRPath string
 	if moduleConfig.DefaultCRPath != "" {
+		if isURL(moduleConfig.DefaultCRPath) {
+			moduleConfig.DefaultCRPath, err = module.DownloadRemoteFileToTempFile(moduleConfig.DefaultCRPath,
+				cmd.opts.Path, "kyma-module-default-cr-*.yaml")
+			if err != nil {
+				return nil, nil, fmt.Errorf("%w,  %w", ErrDefaultCRFetch, err)
+			}
+		}
 		defaultCRPath, err = resolveFilePath(moduleConfig.DefaultCRPath, cmd.opts.Path)
 		if err != nil {
 			return nil, nil, fmt.Errorf("%w,  %w", ErrDefaultCRPathValidation, err)
 		}
 	}
 
-	moduleManifestPath, err := resolveFilePath(moduleConfig.ManifestPath, cmd.opts.Path)
+	var moduleManifestPath string
+	if isURL(moduleConfig.ManifestPath) {
+		moduleConfig.ManifestPath, err = module.DownloadRemoteFileToTempFile(moduleConfig.ManifestPath, cmd.opts.Path,
+			"kyma-module-manifest-*.yaml")
+		if err != nil {
+			return nil, nil, fmt.Errorf("%w,  %w", ErrManifestFetch, err)
+		}
+	}
+	moduleManifestPath, err = resolveFilePath(moduleConfig.ManifestPath, cmd.opts.Path)
 	if err != nil {
 		return nil, nil, fmt.Errorf("%w,  %w", ErrManifestPathValidation, err)
 	}
@@ -631,3 +648,8 @@ func isCrdClusterScoped(crdBytes []byte) bool {
 
 	return crd.Spec.Scope == apiextensions.ClusterScoped
 }
+
+func isURL(s string) bool {
+	u, err := url.Parse(s)
+	return err == nil && u.Scheme != "" && u.Host != ""
+}

cmd/kyma/alpha/create/module/moduleconfig.go

@@ -18,9 +18,9 @@ type Config struct {
 	Name              string                     `yaml:"name"`             // required, the name of the Module
 	Version           string                     `yaml:"version"`          // required, the version of the Module
 	Channel           string                     `yaml:"channel"`          // required, channel that should be used in the ModuleTemplate
-	ManifestPath      string                     `yaml:"manifest"`         // required, reference to the manifests, must be a relative file name.
+	ManifestPath      string                     `yaml:"manifest"`         // required, relative path or remote URL to the manifests.
 	Mandatory         bool                       `yaml:"mandatory"`        // optional, default=false, indicates whether the module is mandatory to be installed on all clusters.
-	DefaultCRPath     string                     `yaml:"defaultCR"`        // optional, reference to a YAML file containing the default CR for the module, must be a relative file name.
+	DefaultCRPath     string                     `yaml:"defaultCR"`        // optional, relative path or remote URL to a YAML file containing the default CR for the module.
 	ResourceName      string                     `yaml:"resourceName"`     // optional, default={NAME}-{CHANNEL}, the name for the ModuleTemplate that will be created
 	Namespace         string                     `yaml:"namespace"`        // optional, default=kcp-system, the namespace where the ModuleTemplate will be deployed
 	Security          string                     `yaml:"security"`         // optional, name of the security scanners config file

cmd/kyma/alpha/create/module/opts.go

@@ -50,7 +50,9 @@ const (
 
 var (
 	ErrChannelValidation       = errors.New("channel validation failed")
+	ErrManifestFetch           = errors.New("remote YAML manifest fetch failed")
 	ErrManifestPathValidation  = errors.New("YAML manifest path validation failed")
+	ErrDefaultCRFetch          = errors.New("remote default CR fetch failed")
 	ErrDefaultCRPathValidation = errors.New("default CR path validation failed")
 	ErrNameValidation          = errors.New("name validation failed")
 	ErrNamespaceValidation     = errors.New("namespace validation failed")

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/go-logr/logr v1.4.1
 	github.com/go-logr/zapr v1.3.0
 	github.com/imdario/mergo v1.0.0
+	github.com/jarcoal/httpmock v1.3.1
 	github.com/kyma-incubator/reconciler v0.0.0-20231215092926-a44f7293b791
 	github.com/kyma-project/hydroform/function v0.0.0-20230831071441-f3501c89bace
 	github.com/kyma-project/hydroform/provision v0.0.0-20230831071441-f3501c89bace

go.sum

@@ -1208,6 +1208,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
+github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY=
@@ -1397,6 +1399,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
+github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
+github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
 github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
 github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=

pkg/module/temp_files.go

@@ -0,0 +1,65 @@
+package module
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+var tempFiles []*os.File
+
+func DownloadRemoteFileToTempFile(url, dir, filenamePattern string) (string, error) {
+	bytes, err := getBytesFromURL(url)
+	if err != nil {
+		return "", fmt.Errorf("failed to download file from %s: %w", url, err)
+	}
+
+	tmpfile, err := os.CreateTemp(dir, filenamePattern)
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp file with pattern %s: %w", filenamePattern, err)
+	}
+	defer tmpfile.Close()
+	tempFiles = append(tempFiles, tmpfile)
+	if _, err := tmpfile.Write(bytes); err != nil {
+		return "", fmt.Errorf("failed to write to temp file %s: %w", tmpfile.Name(), err)
+	}
+
+	return tmpfile.Name(), nil
+}
+
+func DeleteTempFiles() []error {
+	var errors []error
+	for _, file := range tempFiles {
+		err := os.Remove(file.Name())
+		if err != nil {
+			errors = append(errors, err)
+		}
+	}
+	tempFiles = []*os.File{}
+	return errors
+}
+
+func getBytesFromURL(urlString string) ([]byte, error) {
+	url, err := url.Parse(urlString)
+	if err != nil {
+		return nil, fmt.Errorf("parseing url failed for %s: %w", urlString, err)
+	}
+	resp, err := http.Get(url.String())
+	if err != nil {
+		return nil, fmt.Errorf("http GET request failed for %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("bad status for GET request to %s: %s", url, resp.Status)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body from %s: %w", url, err)
+	}
+
+	return data, nil
+}

pkg/module/temp_files_test.go

@@ -0,0 +1,74 @@
+package module
+
+import (
+	"github.com/jarcoal/httpmock"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestDownloadRemoteFileToTempFile(t *testing.T) {
+	t.Parallel()
+
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+	httpmock.RegisterResponder("GET", "https://example.com/manifest.yaml",
+		httpmock.NewBytesResponder(200, []byte("<file-contents>")))
+	defer DeleteTempFiles()
+
+	type args struct {
+		url      string
+		filename string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name: "file download successful",
+			args: args{
+				url:      "https://example.com/manifest.yaml",
+				filename: "manifest-*.yaml",
+			},
+			want:    []byte("<file-contents>"),
+			wantErr: false,
+		},
+		{
+			name: "invalid url results in error",
+			args: args{
+				url:      "invalid-url",
+				filename: "manifest-*.yaml",
+			},
+			want:    []byte{},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := DownloadRemoteFileToTempFile(tt.args.url, os.TempDir(), tt.args.filename)
+
+			if err != nil && !tt.wantErr {
+				t.Errorf("unexpected error occurred: %s", err.Error())
+				return
+			}
+			if err != nil && tt.wantErr {
+				return
+			}
+			if err == nil && tt.wantErr {
+				t.Errorf("expected error did not occur: %s", err.Error())
+				return
+			}
+
+			fileContent, err := os.ReadFile(got)
+			if err != nil {
+				t.Errorf("created file could not be read: %s", err.Error())
+				return
+			}
+			assert.Equalf(t, tt.want, fileContent, "DownloadRemoteFileToTempFile(%v, %v, %v)",
+				tt.args.url, "", tt.args.filename)
+		})
+	}
+}