From 3e96c9202ffad1e4995e287897296f76235becaa Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Mon, 24 Jun 2024 00:05:28 +0300 Subject: [PATCH 1/8] checkpoint --- internal/app/parse.go | 2 ++ internal/datasources/json.go | 17 ++++++++-- internal/datasources/toml.go | 17 ++++++++-- internal/datasources/web.go | 53 ++++++++++++++++++++++++++++++++ internal/datasources/web_test.go | 9 ++++++ internal/datasources/yaml.go | 17 ++++++++-- 6 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 internal/datasources/web.go create mode 100644 internal/datasources/web_test.go diff --git a/internal/app/parse.go b/internal/app/parse.go index b1ea004..5ab1ec1 100644 --- a/internal/app/parse.go +++ b/internal/app/parse.go @@ -97,6 +97,8 @@ func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, err variable = urlWithoutPrefix } return datasources.NewEnvDatasource(variable), nil + case "http", "https": + return datasources.NewWebFileDatasource(url.String()), nil default: return nil, fmt.Errorf("scheme not supported: %s", url.Scheme) } diff --git a/internal/datasources/json.go b/internal/datasources/json.go index 743485c..8748ec8 100644 --- a/internal/datasources/json.go +++ b/internal/datasources/json.go @@ -2,6 +2,7 @@ package datasources import ( "encoding/json" + "io" "os" ) @@ -21,10 +22,22 @@ func (ds *JsonDatasource) Load() (map[string]any, error) { defer file.Close() data := make(map[string]any) - decoder := json.NewDecoder(file) - if err := decoder.Decode(&data); err != nil { + data, err = DecodeJson(file, data) + if err != nil { return nil, err } return data, nil } + +func DecodeJson(r io.Reader, data map[string]any) (map[string]any, error) { + decoder := json.NewDecoder(r) + if err := decoder.Decode(&data); err != nil { + return nil, err + } + return data, nil +} + +func IsJson(content []byte) bool { + return json.Unmarshal(content, &json.RawMessage{}) == nil +} diff --git a/internal/datasources/toml.go b/internal/datasources/toml.go index ef1a0fe..726b5aa 100644 --- a/internal/datasources/toml.go +++ b/internal/datasources/toml.go @@ -1,6 +1,7 @@ package datasources import ( + "io" "os" "github.com/pelletier/go-toml/v2" @@ -22,10 +23,22 @@ func (ds *TomlDatasource) Load() (map[string]any, error) { defer file.Close() data := make(map[string]any) - decoder := toml.NewDecoder(file) - if err := decoder.Decode(&data); err != nil { + data, err = DecodeToml(file, data) + if err != nil { return nil, err } return data, nil } + +func DecodeToml(r io.Reader, data map[string]any) (map[string]any, error) { + decoder := toml.NewDecoder(r) + if err := decoder.Decode(&data); err != nil { + return nil, err + } + return data, nil +} + +func IsToml(content []byte) bool { + return toml.Unmarshal(content, &map[string]interface{}{}) == nil +} diff --git a/internal/datasources/web.go b/internal/datasources/web.go new file mode 100644 index 0000000..b993b4b --- /dev/null +++ b/internal/datasources/web.go @@ -0,0 +1,53 @@ +package datasources + +import ( + "bytes" + "io" + "net/http" +) + +type WebFileDatasource struct { + uri string +} + +func NewWebFileDatasource(uri string) *WebFileDatasource { + return &WebFileDatasource{uri} +} + +func (ds *WebFileDatasource) Load() (map[string]any, error) { + data := make(map[string]any) + + res, err := http.Get(ds.uri) + if err != nil { + return nil, err + } + defer res.Body.Close() + + b, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + // b = []byte(`name = "toml"`) + // b = []byte(`{"name": "json"}`) + // b = []byte(`name: yaml`) + r := bytes.NewReader(b) + + if IsJson(b) { + data, err = DecodeJson(r, data) + if err != nil { + return nil, err + } + } else if IsToml(b) { + data, err = DecodeToml(r, data) + if err != nil { + return nil, err + } + } else if IsYaml(b) { + data, err = DecodeYaml(r, data) + if err != nil { + return nil, err + } + } + + return data, nil +} diff --git a/internal/datasources/web_test.go b/internal/datasources/web_test.go new file mode 100644 index 0000000..6de06df --- /dev/null +++ b/internal/datasources/web_test.go @@ -0,0 +1,9 @@ +package datasources + +import ( + "testing" +) + +func TestWebFileLoad(t *testing.T) { + t.Skip() +} diff --git a/internal/datasources/yaml.go b/internal/datasources/yaml.go index 959bb67..ac60f2e 100644 --- a/internal/datasources/yaml.go +++ b/internal/datasources/yaml.go @@ -1,6 +1,7 @@ package datasources import ( + "io" "os" "gopkg.in/yaml.v3" @@ -22,10 +23,22 @@ func (ds *YamlDatasource) Load() (map[string]any, error) { defer file.Close() data := make(map[string]any) - decoder := yaml.NewDecoder(file) - if err := decoder.Decode(&data); err != nil { + data, err = DecodeYaml(file, data) + if err != nil { return nil, err } return data, nil } + +func DecodeYaml(r io.Reader, data map[string]any) (map[string]any, error) { + decoder := yaml.NewDecoder(r) + if err := decoder.Decode(&data); err != nil { + return nil, err + } + return data, nil +} + +func IsYaml(content []byte) bool { + return yaml.Unmarshal(content, &yaml.Node{}) == nil +} From 836875dae87219fcea42e2478874d08e6053d56b Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Tue, 25 Jun 2024 22:37:24 +0300 Subject: [PATCH 2/8] refactor: decode web file based on mime types --- internal/datasources/json.go | 4 ---- internal/datasources/toml.go | 4 ---- internal/datasources/web.go | 13 +++++++------ internal/datasources/yaml.go | 4 ---- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/internal/datasources/json.go b/internal/datasources/json.go index 8748ec8..f23f985 100644 --- a/internal/datasources/json.go +++ b/internal/datasources/json.go @@ -37,7 +37,3 @@ func DecodeJson(r io.Reader, data map[string]any) (map[string]any, error) { } return data, nil } - -func IsJson(content []byte) bool { - return json.Unmarshal(content, &json.RawMessage{}) == nil -} diff --git a/internal/datasources/toml.go b/internal/datasources/toml.go index 726b5aa..5a36be7 100644 --- a/internal/datasources/toml.go +++ b/internal/datasources/toml.go @@ -38,7 +38,3 @@ func DecodeToml(r io.Reader, data map[string]any) (map[string]any, error) { } return data, nil } - -func IsToml(content []byte) bool { - return toml.Unmarshal(content, &map[string]interface{}{}) == nil -} diff --git a/internal/datasources/web.go b/internal/datasources/web.go index b993b4b..90165de 100644 --- a/internal/datasources/web.go +++ b/internal/datasources/web.go @@ -3,6 +3,7 @@ package datasources import ( "bytes" "io" + "mime" "net/http" ) @@ -23,26 +24,26 @@ func (ds *WebFileDatasource) Load() (map[string]any, error) { } defer res.Body.Close() + ct := res.Header.Get("Content-Type") + mt, _, _ := mime.ParseMediaType(ct) + b, err := io.ReadAll(res.Body) if err != nil { return nil, err } - // b = []byte(`name = "toml"`) - // b = []byte(`{"name": "json"}`) - // b = []byte(`name: yaml`) r := bytes.NewReader(b) - if IsJson(b) { + if mt == "application/json" { data, err = DecodeJson(r, data) if err != nil { return nil, err } - } else if IsToml(b) { + } else if mt == "application/toml" { data, err = DecodeToml(r, data) if err != nil { return nil, err } - } else if IsYaml(b) { + } else if mt == "application/yaml" || mt == "text/yaml" || mt == "text/x-yaml" || mt == "application/x-yaml" { data, err = DecodeYaml(r, data) if err != nil { return nil, err diff --git a/internal/datasources/yaml.go b/internal/datasources/yaml.go index ac60f2e..d255e42 100644 --- a/internal/datasources/yaml.go +++ b/internal/datasources/yaml.go @@ -38,7 +38,3 @@ func DecodeYaml(r io.Reader, data map[string]any) (map[string]any, error) { } return data, nil } - -func IsYaml(content []byte) bool { - return yaml.Unmarshal(content, &yaml.Node{}) == nil -} From 7f9cba76e82b511a68c2fb2bb4713660875ddbd2 Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Tue, 25 Jun 2024 23:23:53 +0300 Subject: [PATCH 3/8] refactor: change ds types to work with reader instead file --- internal/app/parse.go | 29 +++++++++++++++++++++++---- internal/app/parse_test.go | 10 +++++++-- internal/datasources/env_file.go | 18 ++++++----------- internal/datasources/env_file_test.go | 27 ++++++++----------------- internal/datasources/json.go | 24 ++++------------------ internal/datasources/json_test.go | 16 ++++++--------- internal/datasources/toml.go | 24 ++++------------------ internal/datasources/toml_test.go | 16 ++++++--------- internal/datasources/web.go | 9 ++++++--- internal/datasources/yaml.go | 24 ++++------------------ internal/datasources/yaml_test.go | 16 ++++++--------- 11 files changed, 83 insertions(+), 130 deletions(-) diff --git a/internal/app/parse.go b/internal/app/parse.go index 5ab1ec1..f3c4b93 100644 --- a/internal/app/parse.go +++ b/internal/app/parse.go @@ -3,6 +3,7 @@ package app import ( "fmt" "net/url" + "os" "path/filepath" "slices" "strings" @@ -81,13 +82,33 @@ func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, err case "": switch filepath.Ext(urlWithoutPrefix) { case ".yaml", ".yml": - return datasources.NewYamlDatasource(urlWithoutPrefix), nil + f, err := os.Open(urlWithoutPrefix) + if err != nil { + return nil, err + } + // defer f.Close() + return datasources.NewYamlDatasource(f), nil case ".json": - return datasources.NewJsonDatasource(urlWithoutPrefix), nil + f, err := os.Open(urlWithoutPrefix) + if err != nil { + return nil, err + } + // defer f.Close() + return datasources.NewJsonDatasource(f), nil case ".toml": - return datasources.NewTomlDatasource(urlWithoutPrefix), nil + f, err := os.Open(urlWithoutPrefix) + if err != nil { + return nil, err + } + // defer f.Close() + return datasources.NewTomlDatasource(f), nil case ".env": - return datasources.NewEnvFileDatasource(urlWithoutPrefix), nil + f, err := os.Open(urlWithoutPrefix) + if err != nil { + return nil, err + } + // defer f.Close() + return datasources.NewEnvFileDatasource(f), nil default: return nil, fmt.Errorf("unsupported file extension: %s", filepath.Ext(urlWithoutPrefix)) } diff --git a/internal/app/parse_test.go b/internal/app/parse_test.go index 09f4875..c207168 100644 --- a/internal/app/parse_test.go +++ b/internal/app/parse_test.go @@ -13,7 +13,10 @@ import ( func TestCreateYamlDatasourceFromURL(t *testing.T) { a := &App{} - url, err := url.Parse("/tmp/ds.yaml") + tmpDir := t.TempDir() + file, err := os.Create(filepath.Join(tmpDir, "ds.yaml")) + require.NoError(t, err) + url, err := url.Parse(file.Name()) require.NoError(t, err) ds, err := a.createDatasourceFromURL(url) require.NoError(t, err) @@ -22,7 +25,10 @@ func TestCreateYamlDatasourceFromURL(t *testing.T) { func TestCreateJsonDatasourceFromURL(t *testing.T) { a := &App{} - url, err := url.Parse("/tmp/ds.json") + tmpDir := t.TempDir() + file, err := os.Create(filepath.Join(tmpDir, "ds.json")) + require.NoError(t, err) + url, err := url.Parse(file.Name()) require.NoError(t, err) ds, err := a.createDatasourceFromURL(url) require.NoError(t, err) diff --git a/internal/datasources/env_file.go b/internal/datasources/env_file.go index 843f866..1452fec 100644 --- a/internal/datasources/env_file.go +++ b/internal/datasources/env_file.go @@ -2,31 +2,25 @@ package datasources import ( "fmt" - "os" + "io" "github.com/hashicorp/go-envparse" ) type EnvFileDatasource struct { - filepath string + r io.Reader } -func NewEnvFileDatasource(filepath string) *EnvFileDatasource { - return &EnvFileDatasource{filepath} +func NewEnvFileDatasource(r io.Reader) *EnvFileDatasource { + return &EnvFileDatasource{r} } func (ds *EnvFileDatasource) Load() (map[string]any, error) { data := make(map[string]any) - f, err := os.Open(ds.filepath) + env, err := envparse.Parse(ds.r) if err != nil { - return nil, fmt.Errorf("open file %s: %s", ds.filepath, err) - } - defer f.Close() - - env, err := envparse.Parse(f) - if err != nil { - return nil, fmt.Errorf("parse environment variables from file %s: %s", ds.filepath, err) + return nil, fmt.Errorf("parse environment variables: %s", err) } for k, v := range env { diff --git a/internal/datasources/env_file_test.go b/internal/datasources/env_file_test.go index 4be7bab..fe983a3 100644 --- a/internal/datasources/env_file_test.go +++ b/internal/datasources/env_file_test.go @@ -1,34 +1,23 @@ package datasources import ( - "bytes" - "os" "strings" "testing" - "github.com/hashicorp/go-envparse" "github.com/stretchr/testify/require" ) func TestEnvFileLoadFromFile(t *testing.T) { - var expectedData = map[string]any{} - envVars := []string{"key1=value1", "key2=5"} - r := bytes.NewReader([]byte(strings.Join(envVars, "\n"))) - env, err := envparse.Parse(r) - require.NoError(t, err) - for k, v := range env { - expectedData[k] = v - } - - dir := t.TempDir() - file, err := os.CreateTemp(dir, ".env") - require.NoError(t, err) - _, err = file.WriteString(` + envFileData := ` key1=value1 -key2=5`) - require.NoError(t, err) +key2=5` + expectedData := map[string]any{ + "key1": "value1", + "key2": "5", + } + r := strings.NewReader(envFileData) + ds := NewEnvFileDatasource(r) - ds := NewEnvFileDatasource(file.Name()) data, err := ds.Load() require.NoError(t, err) require.Equal(t, expectedData, data) diff --git a/internal/datasources/json.go b/internal/datasources/json.go index f23f985..15c95be 100644 --- a/internal/datasources/json.go +++ b/internal/datasources/json.go @@ -3,35 +3,19 @@ package datasources import ( "encoding/json" "io" - "os" ) type JsonDatasource struct { - filepath string + r io.Reader } -func NewJsonDatasource(filepath string) *JsonDatasource { - return &JsonDatasource{filepath} +func NewJsonDatasource(r io.Reader) *JsonDatasource { + return &JsonDatasource{r} } func (ds *JsonDatasource) Load() (map[string]any, error) { - file, err := os.Open(ds.filepath) - if err != nil { - return nil, err - } - defer file.Close() - data := make(map[string]any) - data, err = DecodeJson(file, data) - if err != nil { - return nil, err - } - - return data, nil -} - -func DecodeJson(r io.Reader, data map[string]any) (map[string]any, error) { - decoder := json.NewDecoder(r) + decoder := json.NewDecoder(ds.r) if err := decoder.Decode(&data); err != nil { return nil, err } diff --git a/internal/datasources/json_test.go b/internal/datasources/json_test.go index 153f209..28d08ef 100644 --- a/internal/datasources/json_test.go +++ b/internal/datasources/json_test.go @@ -1,29 +1,25 @@ package datasources import ( - "os" + "strings" "testing" "github.com/stretchr/testify/require" ) func TestJsonLoad(t *testing.T) { - dir := t.TempDir() - file, err := os.CreateTemp(dir, "ds.json") - require.NoError(t, err) - - _, err = file.WriteString(` + jsonData := ` { "key1": "value1", "key2": 5 -}`) - require.NoError(t, err) - ds := NewJsonDatasource(file.Name()) - +}` expectedData := map[string]any{ "key1": "value1", "key2": float64(5), } + r := strings.NewReader(jsonData) + ds := NewJsonDatasource(r) + data, err := ds.Load() require.NoError(t, err) require.Equal(t, expectedData, data) diff --git a/internal/datasources/toml.go b/internal/datasources/toml.go index 5a36be7..16aed46 100644 --- a/internal/datasources/toml.go +++ b/internal/datasources/toml.go @@ -2,37 +2,21 @@ package datasources import ( "io" - "os" "github.com/pelletier/go-toml/v2" ) type TomlDatasource struct { - filepath string + r io.Reader } -func NewTomlDatasource(filepath string) *TomlDatasource { - return &TomlDatasource{filepath} +func NewTomlDatasource(r io.Reader) *TomlDatasource { + return &TomlDatasource{r} } func (ds *TomlDatasource) Load() (map[string]any, error) { - file, err := os.Open(ds.filepath) - if err != nil { - return nil, err - } - defer file.Close() - data := make(map[string]any) - data, err = DecodeToml(file, data) - if err != nil { - return nil, err - } - - return data, nil -} - -func DecodeToml(r io.Reader, data map[string]any) (map[string]any, error) { - decoder := toml.NewDecoder(r) + decoder := toml.NewDecoder(ds.r) if err := decoder.Decode(&data); err != nil { return nil, err } diff --git a/internal/datasources/toml_test.go b/internal/datasources/toml_test.go index b7e64dd..2df78ae 100644 --- a/internal/datasources/toml_test.go +++ b/internal/datasources/toml_test.go @@ -1,27 +1,23 @@ package datasources import ( - "os" + "strings" "testing" "github.com/stretchr/testify/require" ) func TestTomlLoad(t *testing.T) { - dir := t.TempDir() - file, err := os.CreateTemp(dir, "ds.toml") - require.NoError(t, err) - - _, err = file.WriteString(` + tomlData := ` key1 = "value1" -key2 = 5`) - require.NoError(t, err) - ds := NewTomlDatasource(file.Name()) - +key2 = 5` expectedData := map[string]any{ "key1": "value1", "key2": int64(5), } + r := strings.NewReader(tomlData) + ds := NewTomlDatasource(r) + data, err := ds.Load() require.NoError(t, err) require.Equal(t, expectedData, data) diff --git a/internal/datasources/web.go b/internal/datasources/web.go index 90165de..7fcd9e7 100644 --- a/internal/datasources/web.go +++ b/internal/datasources/web.go @@ -34,17 +34,20 @@ func (ds *WebFileDatasource) Load() (map[string]any, error) { r := bytes.NewReader(b) if mt == "application/json" { - data, err = DecodeJson(r, data) + ds := NewJsonDatasource(r) + data, err = ds.Load() if err != nil { return nil, err } } else if mt == "application/toml" { - data, err = DecodeToml(r, data) + ds := NewTomlDatasource(r) + data, err = ds.Load() if err != nil { return nil, err } } else if mt == "application/yaml" || mt == "text/yaml" || mt == "text/x-yaml" || mt == "application/x-yaml" { - data, err = DecodeYaml(r, data) + ds := NewYamlDatasource(r) + data, err = ds.Load() if err != nil { return nil, err } diff --git a/internal/datasources/yaml.go b/internal/datasources/yaml.go index d255e42..538d0aa 100644 --- a/internal/datasources/yaml.go +++ b/internal/datasources/yaml.go @@ -2,37 +2,21 @@ package datasources import ( "io" - "os" "gopkg.in/yaml.v3" ) type YamlDatasource struct { - filepath string + r io.Reader } -func NewYamlDatasource(filepath string) *YamlDatasource { - return &YamlDatasource{filepath} +func NewYamlDatasource(r io.Reader) *YamlDatasource { + return &YamlDatasource{r} } func (ds *YamlDatasource) Load() (map[string]any, error) { - file, err := os.Open(ds.filepath) - if err != nil { - return nil, err - } - defer file.Close() - data := make(map[string]any) - data, err = DecodeYaml(file, data) - if err != nil { - return nil, err - } - - return data, nil -} - -func DecodeYaml(r io.Reader, data map[string]any) (map[string]any, error) { - decoder := yaml.NewDecoder(r) + decoder := yaml.NewDecoder(ds.r) if err := decoder.Decode(&data); err != nil { return nil, err } diff --git a/internal/datasources/yaml_test.go b/internal/datasources/yaml_test.go index 806506d..2532946 100644 --- a/internal/datasources/yaml_test.go +++ b/internal/datasources/yaml_test.go @@ -1,27 +1,23 @@ package datasources import ( - "os" + "strings" "testing" "github.com/stretchr/testify/require" ) func TestYamlLoad(t *testing.T) { - dir := t.TempDir() - file, err := os.CreateTemp(dir, "ds.yaml") - require.NoError(t, err) - - _, err = file.WriteString(` + yamlData := ` key1: value1 -key2: 5`) - require.NoError(t, err) - ds := NewYamlDatasource(file.Name()) - +key2: 5` expectedData := map[string]any{ "key1": "value1", "key2": 5, } + r := strings.NewReader(yamlData) + ds := NewYamlDatasource(r) + data, err := ds.Load() require.NoError(t, err) require.Equal(t, expectedData, data) From f3d19f310c9849b3222a1626f109608479df47a7 Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Thu, 27 Jun 2024 23:55:48 +0300 Subject: [PATCH 4/8] fix(parse): ensure ds files are closed --- internal/app/parse.go | 35 +++++++++++++++++------------------ internal/app/parse_test.go | 8 ++++---- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/internal/app/parse.go b/internal/app/parse.go index f3c4b93..1dc74e4 100644 --- a/internal/app/parse.go +++ b/internal/app/parse.go @@ -49,10 +49,13 @@ func (a *App) loadDatasources(datasourceUrls []*url.URL, extraData []string, all } for _, url := range datasourceUrls { - ds, err := a.createDatasourceFromURL(url) + ds, f, err := a.createDatasourceFromURL(url) if err != nil { return nil, fmt.Errorf("create datasource %q: %s", url, err) } + if f != nil { + defer f.Close() + } dsData, err := ds.Load() if err != nil { @@ -75,7 +78,7 @@ func (a *App) loadDatasources(datasourceUrls []*url.URL, extraData []string, all return data, nil } -func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, error) { +func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, *os.File, error) { urlWithoutPrefix := strings.TrimPrefix(url.String(), fmt.Sprintf("%s://", url.Scheme)) switch url.Scheme { @@ -84,44 +87,40 @@ func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, err case ".yaml", ".yml": f, err := os.Open(urlWithoutPrefix) if err != nil { - return nil, err + return nil, nil, err } - // defer f.Close() - return datasources.NewYamlDatasource(f), nil + return datasources.NewYamlDatasource(f), f, nil case ".json": f, err := os.Open(urlWithoutPrefix) if err != nil { - return nil, err + return nil, nil, err } - // defer f.Close() - return datasources.NewJsonDatasource(f), nil + return datasources.NewJsonDatasource(f), f, nil case ".toml": f, err := os.Open(urlWithoutPrefix) if err != nil { - return nil, err + return nil, nil, err } - // defer f.Close() - return datasources.NewTomlDatasource(f), nil + return datasources.NewTomlDatasource(f), f, nil case ".env": f, err := os.Open(urlWithoutPrefix) if err != nil { - return nil, err + return nil, nil, err } - // defer f.Close() - return datasources.NewEnvFileDatasource(f), nil + return datasources.NewEnvFileDatasource(f), f, nil default: - return nil, fmt.Errorf("unsupported file extension: %s", filepath.Ext(urlWithoutPrefix)) + return nil, nil, fmt.Errorf("unsupported file extension: %s", filepath.Ext(urlWithoutPrefix)) } case "env": variable := "" if url.Host != "" { variable = urlWithoutPrefix } - return datasources.NewEnvDatasource(variable), nil + return datasources.NewEnvDatasource(variable), nil, nil case "http", "https": - return datasources.NewWebFileDatasource(url.String()), nil + return datasources.NewWebFileDatasource(url.String()), nil, nil default: - return nil, fmt.Errorf("scheme not supported: %s", url.Scheme) + return nil, nil, fmt.Errorf("scheme not supported: %s", url.Scheme) } } diff --git a/internal/app/parse_test.go b/internal/app/parse_test.go index c207168..d3dcb3b 100644 --- a/internal/app/parse_test.go +++ b/internal/app/parse_test.go @@ -18,7 +18,7 @@ func TestCreateYamlDatasourceFromURL(t *testing.T) { require.NoError(t, err) url, err := url.Parse(file.Name()) require.NoError(t, err) - ds, err := a.createDatasourceFromURL(url) + ds, _, err := a.createDatasourceFromURL(url) require.NoError(t, err) require.IsType(t, &datasources.YamlDatasource{}, ds) } @@ -30,7 +30,7 @@ func TestCreateJsonDatasourceFromURL(t *testing.T) { require.NoError(t, err) url, err := url.Parse(file.Name()) require.NoError(t, err) - ds, err := a.createDatasourceFromURL(url) + ds, _, err := a.createDatasourceFromURL(url) require.NoError(t, err) require.IsType(t, &datasources.JsonDatasource{}, ds) } @@ -41,13 +41,13 @@ func TestCreateInvalidDatasourceFromURL(t *testing.T) { // Invalid extension url, err := url.Parse("/tmp/ds.nothing") require.NoError(t, err) - _, err = a.createDatasourceFromURL(url) + _, _, err = a.createDatasourceFromURL(url) require.Error(t, err) // Invalid scheme url, err = url.Parse("nothing:///tmp/ds.yaml") require.NoError(t, err) - _, err = a.createDatasourceFromURL(url) + _, _, err = a.createDatasourceFromURL(url) require.Error(t, err) } From cebdac12105c6a77ac459a1c7d6f2758c294048a Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Fri, 28 Jun 2024 00:04:45 +0300 Subject: [PATCH 5/8] refactor(parse): consolidate ds file open duplicated logic --- internal/app/parse.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/internal/app/parse.go b/internal/app/parse.go index 1dc74e4..7346378 100644 --- a/internal/app/parse.go +++ b/internal/app/parse.go @@ -83,30 +83,18 @@ func (a *App) createDatasourceFromURL(url *url.URL) (datasources.Datasource, *os switch url.Scheme { case "": + f, err := os.Open(urlWithoutPrefix) + if err != nil { + return nil, nil, err + } switch filepath.Ext(urlWithoutPrefix) { case ".yaml", ".yml": - f, err := os.Open(urlWithoutPrefix) - if err != nil { - return nil, nil, err - } return datasources.NewYamlDatasource(f), f, nil case ".json": - f, err := os.Open(urlWithoutPrefix) - if err != nil { - return nil, nil, err - } return datasources.NewJsonDatasource(f), f, nil case ".toml": - f, err := os.Open(urlWithoutPrefix) - if err != nil { - return nil, nil, err - } return datasources.NewTomlDatasource(f), f, nil case ".env": - f, err := os.Open(urlWithoutPrefix) - if err != nil { - return nil, nil, err - } return datasources.NewEnvFileDatasource(f), f, nil default: return nil, nil, fmt.Errorf("unsupported file extension: %s", filepath.Ext(urlWithoutPrefix)) From 4ca82ec57691598faf55aaf5d3516281013a4e4a Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Fri, 28 Jun 2024 00:48:58 +0300 Subject: [PATCH 6/8] test(web): add test to web file load func --- internal/datasources/web_test.go | 41 +++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/internal/datasources/web_test.go b/internal/datasources/web_test.go index 6de06df..6c8e1a1 100644 --- a/internal/datasources/web_test.go +++ b/internal/datasources/web_test.go @@ -1,9 +1,48 @@ package datasources import ( + "fmt" + "net/http" + "net/http/httptest" "testing" + + "github.com/stretchr/testify/require" ) func TestWebFileLoad(t *testing.T) { - t.Skip() + var err error + dsFiles := []string{"ds.json", "ds.toml", "ds.yaml"} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/" + dsFiles[0]: + w.Header().Set("Content-Type", "application/json") + _, err = fmt.Fprint(w, `{"key1": "value1", "key2": "value2"}`) + require.NoError(t, err) + case "/" + dsFiles[1]: + w.Header().Set("Content-Type", "application/toml") + _, err = fmt.Fprint(w, "key1 = \"value1\"\n key2 = \"value2\"") + require.NoError(t, err) + case "/" + dsFiles[2]: + w.Header().Set("Content-Type", "application/yaml") + _, err = fmt.Fprint(w, "key1: value1\nkey2: value2") + require.NoError(t, err) + default: + http.NotFound(w, r) + } + })) + defer ts.Close() + + for _, fileType := range dsFiles { + ds := NewWebFileDatasource(ts.URL + "/" + fileType) + data, err := ds.Load() + require.NoError(t, err) + + expectedData := map[string]any{ + "key1": "value1", + "key2": "value2", + } + + require.Equal(t, data, expectedData) + } } From 172772e9f84c3ed27d3bb18fc57899cbbe29337f Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Fri, 28 Jun 2024 16:39:53 +0300 Subject: [PATCH 7/8] refactor(datasource): tidy up web file ds load func --- internal/datasources/web.go | 41 ++++++++++++++----------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/internal/datasources/web.go b/internal/datasources/web.go index 7fcd9e7..9e43b1c 100644 --- a/internal/datasources/web.go +++ b/internal/datasources/web.go @@ -1,8 +1,7 @@ package datasources import ( - "bytes" - "io" + "fmt" "mime" "net/http" ) @@ -16,8 +15,6 @@ func NewWebFileDatasource(uri string) *WebFileDatasource { } func (ds *WebFileDatasource) Load() (map[string]any, error) { - data := make(map[string]any) - res, err := http.Get(ds.uri) if err != nil { return nil, err @@ -27,31 +24,23 @@ func (ds *WebFileDatasource) Load() (map[string]any, error) { ct := res.Header.Get("Content-Type") mt, _, _ := mime.ParseMediaType(ct) - b, err := io.ReadAll(res.Body) + var targetDs Datasource + + switch mt { + case "application/json": + targetDs = NewJsonDatasource(res.Body) + case "application/toml": + targetDs = NewTomlDatasource(res.Body) + case "application/yaml", "text/yaml", "text/x-yaml", "application/x-yaml": + targetDs = NewYamlDatasource(res.Body) + default: + return nil, fmt.Errorf("unsupported content type: %s", mt) + } + + data, err := targetDs.Load() if err != nil { return nil, err } - r := bytes.NewReader(b) - - if mt == "application/json" { - ds := NewJsonDatasource(r) - data, err = ds.Load() - if err != nil { - return nil, err - } - } else if mt == "application/toml" { - ds := NewTomlDatasource(r) - data, err = ds.Load() - if err != nil { - return nil, err - } - } else if mt == "application/yaml" || mt == "text/yaml" || mt == "text/x-yaml" || mt == "application/x-yaml" { - ds := NewYamlDatasource(r) - data, err = ds.Load() - if err != nil { - return nil, err - } - } return data, nil } From 1f224506dd8b65a9b8277e95dc6364ad7c963dc6 Mon Sep 17 00:00:00 2001 From: Aviad Haham Date: Fri, 28 Jun 2024 16:41:35 +0300 Subject: [PATCH 8/8] change uri to url in web file ds --- internal/datasources/web.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/datasources/web.go b/internal/datasources/web.go index 9e43b1c..f9e0add 100644 --- a/internal/datasources/web.go +++ b/internal/datasources/web.go @@ -7,15 +7,15 @@ import ( ) type WebFileDatasource struct { - uri string + url string } -func NewWebFileDatasource(uri string) *WebFileDatasource { - return &WebFileDatasource{uri} +func NewWebFileDatasource(url string) *WebFileDatasource { + return &WebFileDatasource{url} } func (ds *WebFileDatasource) Load() (map[string]any, error) { - res, err := http.Get(ds.uri) + res, err := http.Get(ds.url) if err != nil { return nil, err }