feat(datasource): support external datasource urls

internal/app/parse.go

@@ -3,6 +3,7 @@ package app
 import (
 	"fmt"
 	"net/url"
+	"os"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -48,10 +49,13 @@ func (a *App) loadDatasources(datasourceUrls []*url.URL, extraData []string, all
 	}
 
 	for _, url := range datasourceUrls {
-		ds, err := a.createDatasourceFromURL(url)
+		ds, f, err := a.createDatasourceFromURL(url)
 		if err != nil {
 			return nil, fmt.Errorf("create datasource %q: %s", url, err)
 		}
+		if f != nil {
+			defer f.Close()
+		}
 
 		dsData, err := ds.Load()
 		if err != nil {
@@ -74,31 +78,37 @@ func (a *App) loadDatasources(datasourceUrls []*url.URL, extraData []string, all
 	return data, nil
 }
 
-func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, error) {
+func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, *os.File, error) {
 	urlWithoutPrefix := strings.TrimPrefix(url.String(), fmt.Sprintf("%s://", url.Scheme))
 
 	switch url.Scheme {
 	case "":
+		f, err := os.Open(urlWithoutPrefix)
+		if err != nil {
+			return nil, nil, err
+		}
 		switch filepath.Ext(urlWithoutPrefix) {
 		case ".yaml", ".yml":
-			return datasources.NewYamlDatasource(urlWithoutPrefix), nil
+			return datasources.NewYamlDatasource(f), f, nil
 		case ".json":
-			return datasources.NewJsonDatasource(urlWithoutPrefix), nil
+			return datasources.NewJsonDatasource(f), f, nil
 		case ".toml":
-			return datasources.NewTomlDatasource(urlWithoutPrefix), nil
+			return datasources.NewTomlDatasource(f), f, nil
 		case ".env":
-			return datasources.NewEnvFileDatasource(urlWithoutPrefix), nil
+			return datasources.NewEnvFileDatasource(f), f, nil
 		default:
-			return nil, fmt.Errorf("unsupported file extension: %s", filepath.Ext(urlWithoutPrefix))
+			return nil, nil, fmt.Errorf("unsupported file extension: %s", filepath.Ext(urlWithoutPrefix))
 		}
 	case "env":
 		variable := ""
 		if url.Host != "" {
 			variable = urlWithoutPrefix
 		}
-		return datasources.NewEnvDatasource(variable), nil
+		return datasources.NewEnvDatasource(variable), nil, nil
+	case "http", "https":
+		return datasources.NewWebFileDatasource(url.String()), nil, nil
 	default:
-		return nil, fmt.Errorf("scheme not supported: %s", url.Scheme)
+		return nil, nil, fmt.Errorf("scheme not supported: %s", url.Scheme)
 	}
 }
 

internal/app/parse_test.go

@@ -13,18 +13,24 @@ import (
 
 func TestCreateYamlDatasourceFromURL(t *testing.T) {
 	a := &App{}
-	url, err := url.Parse("/tmp/ds.yaml")
+	tmpDir := t.TempDir()
+	file, err := os.Create(filepath.Join(tmpDir, "ds.yaml"))
+	require.NoError(t, err)
+	url, err := url.Parse(file.Name())
 	require.NoError(t, err)
-	ds, err := a.createDatasourceFromURL(url)
+	ds, _, err := a.createDatasourceFromURL(url)
 	require.NoError(t, err)
 	require.IsType(t, &datasources.YamlDatasource{}, ds)
 }
 
 func TestCreateJsonDatasourceFromURL(t *testing.T) {
 	a := &App{}
-	url, err := url.Parse("/tmp/ds.json")
+	tmpDir := t.TempDir()
+	file, err := os.Create(filepath.Join(tmpDir, "ds.json"))
+	require.NoError(t, err)
+	url, err := url.Parse(file.Name())
 	require.NoError(t, err)
-	ds, err := a.createDatasourceFromURL(url)
+	ds, _, err := a.createDatasourceFromURL(url)
 	require.NoError(t, err)
 	require.IsType(t, &datasources.JsonDatasource{}, ds)
 }
@@ -35,13 +41,13 @@ func TestCreateInvalidDatasourceFromURL(t *testing.T) {
 	// Invalid extension
 	url, err := url.Parse("/tmp/ds.nothing")
 	require.NoError(t, err)
-	_, err = a.createDatasourceFromURL(url)
+	_, _, err = a.createDatasourceFromURL(url)
 	require.Error(t, err)
 
 	// Invalid scheme
 	url, err = url.Parse("nothing:///tmp/ds.yaml")
 	require.NoError(t, err)
-	_, err = a.createDatasourceFromURL(url)
+	_, _, err = a.createDatasourceFromURL(url)
 	require.Error(t, err)
 }
 

internal/datasources/env_file.go

@@ -2,31 +2,25 @@ package datasources
 
 import (
 	"fmt"
-	"os"
+	"io"
 
 	"github.com/hashicorp/go-envparse"
 )
 
 type EnvFileDatasource struct {
-	filepath string
+	r io.Reader
 }
 
-func NewEnvFileDatasource(filepath string) *EnvFileDatasource {
-	return &EnvFileDatasource{filepath}
+func NewEnvFileDatasource(r io.Reader) *EnvFileDatasource {
+	return &EnvFileDatasource{r}
 }
 
 func (ds *EnvFileDatasource) Load() (map[string]any, error) {
 	data := make(map[string]any)
 
-	f, err := os.Open(ds.filepath)
+	env, err := envparse.Parse(ds.r)
 	if err != nil {
-		return nil, fmt.Errorf("open file %s: %s", ds.filepath, err)
-	}
-	defer f.Close()
-
-	env, err := envparse.Parse(f)
-	if err != nil {
-		return nil, fmt.Errorf("parse environment variables from file %s: %s", ds.filepath, err)
+		return nil, fmt.Errorf("parse environment variables: %s", err)
 	}
 
 	for k, v := range env {

internal/datasources/env_file_test.go

@@ -1,34 +1,23 @@
 package datasources
 
 import (
-	"bytes"
-	"os"
 	"strings"
 	"testing"
 
-	"github.com/hashicorp/go-envparse"
 	"github.com/stretchr/testify/require"
 )
 
 func TestEnvFileLoadFromFile(t *testing.T) {
-	var expectedData = map[string]any{}
-	envVars := []string{"key1=value1", "key2=5"}
-	r := bytes.NewReader([]byte(strings.Join(envVars, "\n")))
-	env, err := envparse.Parse(r)
-	require.NoError(t, err)
-	for k, v := range env {
-		expectedData[k] = v
-	}
-
-	dir := t.TempDir()
-	file, err := os.CreateTemp(dir, ".env")
-	require.NoError(t, err)
-	_, err = file.WriteString(`
+	envFileData := `
 key1=value1
-key2=5`)
-	require.NoError(t, err)
+key2=5`
+	expectedData := map[string]any{
+		"key1": "value1",
+		"key2": "5",
+	}
+	r := strings.NewReader(envFileData)
+	ds := NewEnvFileDatasource(r)
 
-	ds := NewEnvFileDatasource(file.Name())
 	data, err := ds.Load()
 	require.NoError(t, err)
 	require.Equal(t, expectedData, data)

internal/datasources/json.go

@@ -2,29 +2,22 @@ package datasources
 
 import (
 	"encoding/json"
-	"os"
+	"io"
 )
 
 type JsonDatasource struct {
-	filepath string
+	r io.Reader
 }
 
-func NewJsonDatasource(filepath string) *JsonDatasource {
-	return &JsonDatasource{filepath}
+func NewJsonDatasource(r io.Reader) *JsonDatasource {
+	return &JsonDatasource{r}
 }
 
 func (ds *JsonDatasource) Load() (map[string]any, error) {
-	file, err := os.Open(ds.filepath)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
 	data := make(map[string]any)
-	decoder := json.NewDecoder(file)
+	decoder := json.NewDecoder(ds.r)
 	if err := decoder.Decode(&data); err != nil {
 		return nil, err
 	}
-
 	return data, nil
 }

internal/datasources/json_test.go

@@ -1,29 +1,25 @@
 package datasources
 
 import (
-	"os"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestJsonLoad(t *testing.T) {
-	dir := t.TempDir()
-	file, err := os.CreateTemp(dir, "ds.json")
-	require.NoError(t, err)
-
-	_, err = file.WriteString(`
+	jsonData := `
 {
 	"key1": "value1",
 	"key2": 5
-}`)
-	require.NoError(t, err)
-	ds := NewJsonDatasource(file.Name())
-
+}`
 	expectedData := map[string]any{
 		"key1": "value1",
 		"key2": float64(5),
 	}
+	r := strings.NewReader(jsonData)
+	ds := NewJsonDatasource(r)
+
 	data, err := ds.Load()
 	require.NoError(t, err)
 	require.Equal(t, expectedData, data)

internal/datasources/toml.go

@@ -1,31 +1,24 @@
 package datasources
 
 import (
-	"os"
+	"io"
 
 	"github.com/pelletier/go-toml/v2"
 )
 
 type TomlDatasource struct {
-	filepath string
+	r io.Reader
 }
 
-func NewTomlDatasource(filepath string) *TomlDatasource {
-	return &TomlDatasource{filepath}
+func NewTomlDatasource(r io.Reader) *TomlDatasource {
+	return &TomlDatasource{r}
 }
 
 func (ds *TomlDatasource) Load() (map[string]any, error) {
-	file, err := os.Open(ds.filepath)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
 	data := make(map[string]any)
-	decoder := toml.NewDecoder(file)
+	decoder := toml.NewDecoder(ds.r)
 	if err := decoder.Decode(&data); err != nil {
 		return nil, err
 	}
-
 	return data, nil
 }

internal/datasources/toml_test.go

@@ -1,27 +1,23 @@
 package datasources
 
 import (
-	"os"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestTomlLoad(t *testing.T) {
-	dir := t.TempDir()
-	file, err := os.CreateTemp(dir, "ds.toml")
-	require.NoError(t, err)
-
-	_, err = file.WriteString(`
+	tomlData := `
 key1 = "value1"
-key2 = 5`)
-	require.NoError(t, err)
-	ds := NewTomlDatasource(file.Name())
-
+key2 = 5`
 	expectedData := map[string]any{
 		"key1": "value1",
 		"key2": int64(5),
 	}
+	r := strings.NewReader(tomlData)
+	ds := NewTomlDatasource(r)
+
 	data, err := ds.Load()
 	require.NoError(t, err)
 	require.Equal(t, expectedData, data)

internal/datasources/web.go

@@ -0,0 +1,46 @@
+package datasources
+
+import (
+	"fmt"
+	"mime"
+	"net/http"
+)
+
+type WebFileDatasource struct {
+	url string
+}
+
+func NewWebFileDatasource(url string) *WebFileDatasource {
+	return &WebFileDatasource{url}
+}
+
+func (ds *WebFileDatasource) Load() (map[string]any, error) {
+	res, err := http.Get(ds.url)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	ct := res.Header.Get("Content-Type")
+	mt, _, _ := mime.ParseMediaType(ct)
+
+	var targetDs Datasource
+
+	switch mt {
+	case "application/json":
+		targetDs = NewJsonDatasource(res.Body)
+	case "application/toml":
+		targetDs = NewTomlDatasource(res.Body)
+	case "application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml":
+		targetDs = NewYamlDatasource(res.Body)
+	default:
+		return nil, fmt.Errorf("unsupported content type: %s", mt)
+	}
+
+	data, err := targetDs.Load()
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}