Multiple Files Support - Part 3

.golangci.yml

@@ -3,6 +3,7 @@ run:
   skip-dirs:
     - pkg/worker
     - cmd/webworker
+    - web
 issues:
   exclude-rules:
     - path: 'internal/gowasm/(.+)\.go'

cmd/playground/main.go

@@ -11,8 +11,8 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/x1unix/foundation/app"
 	"github.com/x1unix/go-playground/internal/analyzer"
-	"github.com/x1unix/go-playground/internal/compiler"
-	"github.com/x1unix/go-playground/internal/compiler/storage"
+	"github.com/x1unix/go-playground/internal/builder"
+	"github.com/x1unix/go-playground/internal/builder/storage"
 	"github.com/x1unix/go-playground/internal/config"
 	"github.com/x1unix/go-playground/internal/langserver"
 	"github.com/x1unix/go-playground/internal/langserver/webutil"
@@ -39,7 +39,7 @@ func main() {
 	analyzer.SetLogger(logger)
 	defer logger.Sync() //nolint:errcheck
 
-	goRoot, err := compiler.GOROOT()
+	goRoot, err := builder.GOROOT()
 	if err != nil {
 		logger.Fatal("Failed to find GOROOT environment variable value", zap.Error(err))
 	}
@@ -58,24 +58,28 @@ func start(goRoot string, logger *zap.Logger, cfg *config.Config) error {
 		return fmt.Errorf("failed to read packages file %q: %s", cfg.Build.PackagesFile, err)
 	}
 
-	store, err := storage.NewLocalStorage(logger.Sugar(), cfg.Build.BuildDir)
+	store, err := storage.NewLocalStorage(logger, cfg.Build.BuildDir)
 	if err != nil {
 		return err
 	}
 
 	ctx, _ := app.GetApplicationContext()
 	wg := &sync.WaitGroup{}
-	go store.StartCleaner(ctx, cfg.Build.CleanupInterval, nil)
 
 	// Initialize services
 	playgroundClient := goplay.NewClient(cfg.Playground.PlaygroundURL, goplay.DefaultUserAgent,
 		cfg.Playground.ConnectTimeout)
-	buildCfg := compiler.BuildEnvironmentConfig{
+	buildCfg := builder.BuildEnvironmentConfig{
+		KeepGoModCache:               cfg.Build.SkipModuleCleanup,
 		IncludedEnvironmentVariables: osutil.SelectEnvironmentVariables(cfg.Build.BypassEnvVarsList...),
 	}
 	logger.Debug("Loaded list of environment variables used by compiler",
 		zap.Any("vars", buildCfg.IncludedEnvironmentVariables))
-	buildSvc := compiler.NewBuildService(zap.S(), buildCfg, store)
+	buildSvc := builder.NewBuildService(zap.L(), buildCfg, store)
+
+	// Start cleanup service
+	cleanupSvc := builder.NewCleanupDispatchService(zap.L(), cfg.Build.CleanupInterval, buildSvc, store)
+	go cleanupSvc.Start(ctx)
 
 	// Initialize API endpoints
 	r := mux.NewRouter()

go.mod

@@ -9,14 +9,14 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/pkg/errors v0.8.1
 	github.com/samber/lo v1.38.1
-	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.2
 	github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5
 	github.com/traefik/yaegi v0.15.1
 	github.com/x1unix/foundation v1.0.0
+	go.uber.org/mock v0.4.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
-	golang.org/x/mod v0.10.0
+	golang.org/x/mod v0.14.0
 	golang.org/x/sync v0.2.0
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
 )
@@ -25,9 +25,7 @@ require (
 	github.com/benbjohnson/clock v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getsentry/sentry-go v0.13.0 // indirect
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect

go.sum

@@ -4,7 +4,6 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -14,8 +13,6 @@ github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -28,13 +25,8 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
 github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -56,6 +48,8 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
 go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
@@ -67,8 +61,8 @@ golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAb
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
-golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

internal/builder/check.go

@@ -0,0 +1,48 @@
+package builder
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/x1unix/go-playground/pkg/goplay"
+)
+
+const (
+	maxPathDepth = 5
+	maxFileCount = 12
+)
+
+func checkFileEntries(entries map[string][]byte) error {
+	if len(entries) == 0 {
+		return newBuildError("no buildable Go source files")
+	}
+
+	if len(entries) > maxFileCount {
+		return newBuildError("too many files (max: %d)", maxFileCount)
+	}
+
+	for name, contents := range entries {
+		if len(bytes.TrimSpace(contents)) == 0 {
+			return newBuildError("file %s is empty", name)
+		}
+
+		if err := checkFilePath(name); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func checkFilePath(fpath string) error {
+	if err := goplay.ValidateFilePath(fpath, true); err != nil {
+		return newBuildError(err.Error())
+	}
+
+	pathDepth := strings.Count(fpath, "/")
+	if pathDepth > maxPathDepth {
+		return newBuildError("file path is too deep: %s", fpath)
+	}
+
+	return nil
+}

internal/builder/cleanup.go

@@ -0,0 +1,87 @@
+package builder
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/tevino/abool"
+	"go.uber.org/zap"
+)
+
+type Cleaner interface {
+	CleanJobName() string
+	Clean(ctx context.Context) error
+}
+
+// CleanupDispatchService calls cleanup entries after periodical interval of time.
+type CleanupDispatchService struct {
+	isRunning abool.AtomicBool
+	logger    *zap.Logger
+	interval  time.Duration
+	cleaners  []Cleaner
+}
+
+func NewCleanupDispatchService(logger *zap.Logger, interval time.Duration, cleaners ...Cleaner) *CleanupDispatchService {
+	return &CleanupDispatchService{
+		logger:   logger.Named("cleanup"),
+		interval: interval,
+		cleaners: cleaners,
+	}
+}
+
+func (c *CleanupDispatchService) Start(ctx context.Context) {
+	t := time.NewTicker(c.interval)
+	defer t.Stop()
+
+	c.logger.Info("started cleanup service", zap.Duration("interval", c.interval))
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			go c.dispatchCleanup(ctx)
+		}
+	}
+}
+
+func (c *CleanupDispatchService) dispatchCleanup(ctx context.Context) {
+	if c.isRunning.IsSet() {
+		c.logger.Info("previous job not finished yet, skip")
+		return
+	}
+
+	c.isRunning.Set()
+	defer c.isRunning.UnSet()
+
+	c.logger.Info("starting cleanup job")
+	startTime := time.Now()
+	jobCtx, cancelFn := context.WithTimeout(ctx, c.interval)
+	defer cancelFn()
+
+	wg := new(sync.WaitGroup)
+	for _, cleaner := range c.cleaners {
+		wg.Add(1)
+		go func(cleaner Cleaner) {
+			defer wg.Done()
+			if err := cleaner.Clean(jobCtx); err != nil {
+				if errors.Is(err, context.Canceled) {
+					return
+				}
+				c.logger.Error(
+					"cleaner returned an error",
+					zap.Error(err), zap.String("cleaner", cleaner.CleanJobName()),
+				)
+			}
+		}(cleaner)
+	}
+
+	wg.Wait()
+	duration := time.Since(startTime)
+	if duration > c.interval {
+		c.logger.Warn("cleanup job took too long!", zap.Duration("duration", duration))
+		return
+	}
+	c.logger.Info("cleanup job finished", zap.Duration("duration", duration))
+}