feat: support vendoring (#33)

.golangci.yml

@@ -38,4 +38,7 @@ issues:
   exclude-rules:
     - path: module/client.go
       linters:
-        - noctx
+        - noctx
+    - path: internal/modules/vendor.go
+      linters:
+        - prealloc

README.md

@@ -63,7 +63,7 @@ The application will need to be able to create files and folders in this path.
 
 **--log.format** FORMAT, **$LOG_FORMAT** *(Default: "logfmt")*
 
-Specify the format of logs. Supported formats: 'logfmt', 'json'.
+Specify the format of logs. Supported formats: 'logfmt', 'json', 'console'.
 
 **--log.level** LEVEL, **$LOG_LEVEL** *(Default: "info")*
 
@@ -199,9 +199,9 @@ func New(ctx context.Context, cfg *Config, info types.Info, ui types.UI) (io.Clo
 
 #### Dependencies
 
-All dependencies must vendored except for `github.com/glasslabs/looking-glass/module/types`. 
+All dependencies must be vendored except for `github.com/glasslabs/looking-glass/module/types`. 
 If you still wish to use Go Modules for dependency management, you should run `go mod vendor` to 
-vendor your dependencies and commit your `vendor` folder to git.
+vendor your dependencies and commit your `vendor/modules.txt` to git.
 
 More information about vendoring can be found in the [Go Module Reference](https://golang.org/ref/mod#vendoring).
 
@@ -210,4 +210,4 @@ More information about vendoring can be found in the [Go Module Reference](https
 This is very much a work in progress and under active development. The immediate list of
 things to do is below:
 
-* Installation script and assets for Raspberry Pi
+* Localisation

example/simple/config.yaml

@@ -16,10 +16,12 @@ modules:
       appId: {{ .Secrets.weather.appId }}
       units: metric
   - name: simple-calendar
-    path: calendar
+    path: github.com/glasslabs/calendar
+    version: latest
     position: top:right
     config:
-      maxDays: 30
+      timezone: Africa/Johannesburg
+      maxDays: 5
       calendars:
         - url: https://www.calendarlabs.com/ical-calendar/ics/68/South_Africa_Holidays.ics
         - url: {{ .Secrets.caelndar.myCalendar }}

internal/modules/vendor.go

@@ -0,0 +1,34 @@
+package modules
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/mod/module"
+)
+
+// Dependencies reads module dependencies from a vendored modules file.
+func Dependencies(path string) ([]module.Version, error) {
+	path = filepath.Join(path, "vendor", "modules.txt")
+	if _, err := os.Stat(path); err != nil {
+		return nil, nil
+	}
+
+	vendor, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var deps []module.Version
+	for _, line := range strings.Split(strings.TrimSpace(string(vendor)), "\n") {
+		parts := strings.Fields(line)
+		if len(parts) < 3 || parts[0] != "#" {
+			continue
+		}
+
+		deps = append(deps, module.Version{Path: parts[1], Version: parts[2]})
+	}
+
+	return deps, nil
+}

module/module.go

@@ -14,14 +14,14 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/glasslabs/looking-glass/internal/modules"
 	stypes "github.com/glasslabs/looking-glass/module/internal/types"
 	"github.com/glasslabs/looking-glass/module/types"
 	"github.com/hamba/logger/v2"
 	"github.com/hamba/logger/v2/ctx"
 	"github.com/traefik/yaegi/interp"
 	"github.com/traefik/yaegi/stdlib"
 	"github.com/traefik/yaegi/stdlib/unsafe"
-	"golang.org/x/mod/module"
 	"gopkg.in/yaml.v3"
 )
 
@@ -141,62 +141,88 @@ func (s Service) debug(msg string, ctx ...logger.Field) {
 
 // Extract downloads and extracts a module into the module path.
 func (s Service) Extract(desc Descriptor) error {
-	if desc.Version == "" {
-		s.debug("module has no version, ignoring", ctx.Str("path", desc.Path))
+	path, err := s.extract(desc.Path, desc.Version)
+	if err != nil {
+		return err
+	}
+
+	deps, err := modules.Dependencies(path)
+	if err != nil {
+		return fmt.Errorf("could not read vendor modules: %w", err)
+	}
+
+	for _, dep := range deps {
+		// Exclude ourselves, we are already included.
+		if dep.Path == "github.com/glasslabs/looking-glass" {
+			continue
+		}
+
+		s.debug("extracting dependency", ctx.Str("module", dep.Path), ctx.Str("ver", dep.Version))
+
+		if _, err = s.extract(dep.Path, dep.Version); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s Service) extract(path, ver string) (string, error) {
+	if ver == "" {
+		s.debug("module has no version, ignoring", ctx.Str("path", path))
 
 		// User is not expecting us to extract. Nothing to do.
-		return nil
+		return "", nil
 	}
-	m, err := s.c.Version(desc.Path, desc.Version)
+	m, err := s.c.Version(path, ver)
 	if err != nil {
-		return err
+		return "", err
 	}
 	s.debug("module version resolved", ctx.Str("module", m.Path), ctx.Str("ver", m.Version))
 
-	modPath := filepath.Join(s.path, srcPath, desc.Path)
+	modPath := filepath.Join(s.path, srcPath, path)
 	markerPath := filepath.Join(modPath, markerFile)
 	if _, err = os.Stat(modPath); err == nil {
 		// This might be a user controlled path, check for the marker.
 		if _, err = os.Stat(markerPath); err != nil {
 			s.debug("path seems to be a user module path", ctx.Str("path", modPath))
 			// Not our path or something we cannot touch.
-			return nil
+			return "", nil
 		}
 		if ver, err := os.ReadFile(markerPath); err == nil && m.Version == string(ver) {
 			s.debug("module is at correct version", ctx.Str("path", modPath))
 			// The correct version is already extracted. Nothing to do.
-			return nil
+			return modPath, nil
 		}
 
 		// The path exists but is the wrong version, remove it.
 		s.debug("cleaning module path", ctx.Str("path", modPath))
 		if err = os.RemoveAll(modPath); err != nil {
-			return fmt.Errorf("could not remove old module: %w", err)
+			return "", fmt.Errorf("could not remove old module: %w", err)
 		}
 	}
 
-	s.debug("extracting module", ctx.Str("path", m.Path), ctx.Str("ver", m.Version))
+	s.debug("extracting module", ctx.Str("path", path), ctx.Str("ver", m.Version))
 	z, err := s.c.Download(m)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer func() {
 		_ = z.Close()
 	}()
 	if err = os.MkdirAll(modPath, 0750); err != nil {
-		return fmt.Errorf("could not create module path %q: %w", modPath, err)
+		return "", fmt.Errorf("could not create module path %q: %w", modPath, err)
 	}
-	if err = s.unzip(z, m, modPath); err != nil {
-		return fmt.Errorf("could not extract module: %w", err)
+	if err = s.unzip(z, path, m.Version, modPath); err != nil {
+		return "", fmt.Errorf("could not extract module: %w", err)
 	}
 	if err = os.WriteFile(markerPath, []byte(m.Version), 0440); err != nil {
-		return fmt.Errorf("could not write module marker: %w", err)
+		return "", fmt.Errorf("could not write module marker: %w", err)
 	}
 
-	return nil
+	return modPath, nil
 }
 
-func (s Service) unzip(r io.Reader, m module.Version, path string) error {
+func (s Service) unzip(r io.Reader, modPath, version, path string) error {
 	var buf bytes.Buffer
 	size, err := io.Copy(&buf, r)
 	if err != nil {
@@ -208,7 +234,7 @@ func (s Service) unzip(r io.Reader, m module.Version, path string) error {
 	if err != nil {
 		return err
 	}
-	prefix := fmt.Sprintf("%s@%s/", m.Path, m.Version)
+	prefix := fmt.Sprintf("%s@%s/", modPath, version)
 	for _, zf := range z.File {
 		if !strings.HasPrefix(zf.Name, prefix) {
 			return fmt.Errorf("unexpected file name %s", zf.Name)

module/module_test.go

@@ -188,6 +188,41 @@ func TestService_Extract(t *testing.T) {
 	assert.Equal(t, "v0.2.0", string(b))
 }
 
+func TestService_ExtractWithVendor(t *testing.T) {
+	dir, err := os.MkdirTemp("./", "extract-test")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	b, err := os.ReadFile("../testdata/outer-module@v0.1.0.zip")
+	require.NoError(t, err)
+	r1 := io.NopCloser(bytes.NewReader(b))
+	b, err = os.ReadFile("../testdata/module@v0.2.0.zip")
+	require.NoError(t, err)
+	r2 := io.NopCloser(bytes.NewReader(b))
+	c := &MockClient{}
+	c.On("Version", "outer-module", "v0.1.0").Twice().Return(mod.Version{Path: "outer-module", Version: "v0.1.0"}, nil)
+	c.On("Version", "test-module", "v0.2.0").Once().Return(mod.Version{Path: "test-module", Version: "v0.2.0"}, nil)
+	c.On("Download", mod.Version{Path: "outer-module", Version: "v0.1.0"}).Once().Return(r1, nil)
+	c.On("Download", mod.Version{Path: "test-module", Version: "v0.2.0"}).Once().Return(r2, nil)
+
+	svc, err := module.NewService(dir, c)
+	require.NoError(t, err)
+
+	err = svc.Extract(module.Descriptor{
+		Name:    "test",
+		Path:    "outer-module",
+		Version: "v0.1.0",
+	})
+	require.NoError(t, err)
+
+	b, _ = os.ReadFile(filepath.Join(dir, "src/test-module/main.go"))
+	assert.Equal(t, "test-module\n", string(b))
+	b, _ = os.ReadFile(filepath.Join(dir, "src/test-module/.looking-glass"))
+	assert.Equal(t, "v0.2.0", string(b))
+}
+
 func TestService_ExtractLeavesUserModule(t *testing.T) {
 	dir, err := os.MkdirTemp("./", "extract-test")
 	require.NoError(t, err)