Add Hugo Modules

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082

benchbep.sh

@@ -1,2 +1 @@
-gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench
-benchcmp -best 0.bench 1.bench
+gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"

cache/filecache/filecache.go

@@ -319,7 +319,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 		var cfs afero.Fs
 
 		if v.isResourceDir {
-			cfs = p.BaseFs.Resources.Fs
+			cfs = p.BaseFs.ResourcesCache
 		} else {
 			cfs = fs
 		}

cache/filecache/filecache_config_test.go

@@ -20,6 +20,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gohugoio/hugo/modules"
+
+	"github.com/gohugoio/hugo/langs"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/config"
@@ -55,8 +59,9 @@ dir = "/path/to/c3"
 
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
+	cfg, err := configFromString(configStr)
 	assert.NoError(err)
+
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
@@ -76,6 +81,23 @@ dir = "/path/to/c3"
 
 }
 
+func configFromString(configStr string) (config.Provider, error) {
+	cfg, err := config.FromConfigString(configStr, "toml")
+	if err != nil {
+		return nil, err
+	}
+	if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {
+		return nil, err
+	}
+	mod, err := modules.CreateProjectModule(cfg)
+	if err != nil {
+		return nil, err
+	}
+	cfg.Set("allModules", modules.Modules{mod})
+
+	return cfg, nil
+}
+
 func TestDecodeConfigIgnoreCache(t *testing.T) {
 	t.Parallel()
 
@@ -103,8 +125,7 @@ dir = "/path/to/c3"
 
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
+	cfg, err := configFromString(configStr)
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
@@ -181,8 +202,7 @@ dir = "/"
 		configStr = strings.Replace(configStr, "/", "c:\\\\", 1)
 	}
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
+	cfg, err := configFromString(configStr)
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
@@ -203,5 +223,12 @@ func newTestConfig() *viper.Viper {
 	cfg.Set("archetypeDir", "archetypes")
 	cfg.Set("assetDir", "assets")
 
+	langs.LoadLanguageSettings(cfg, nil)
+	mod, err := modules.CreateProjectModule(cfg)
+	if err != nil {
+		panic(err)
+	}
+	cfg.Set("allModules", modules.Modules{mod})
+
 	return cfg
 }

cache/filecache/filecache_pruner_test.go

@@ -18,7 +18,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 
@@ -54,7 +53,7 @@ maxAge = "200ms"
 dir = ":resourceDir/_gen"
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
+	cfg, err := configFromString(configStr)
 	assert.NoError(err)
 
 	for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} {

cache/filecache/filecache_test.go

@@ -26,7 +26,6 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/common/hugio"
-	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/hugofs"
@@ -83,8 +82,7 @@ dir = ":cacheDir/c"
 		configStr = replacer.Replace(configStr)
 		configStr = strings.Replace(configStr, "\\", winPathSep, -1)
 
-		cfg, err := config.FromConfigString(configStr, "toml")
-		assert.NoError(err)
+		cfg, err := configFromString(configStr)
 
 		fs := hugofs.NewFrom(osfs, cfg)
 		p, err := helpers.NewPathSpec(fs, cfg)
@@ -207,8 +205,7 @@ dir = "/cache/c"
 
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
+	cfg, err := configFromString(configStr)
 	fs := hugofs.NewMem(cfg)
 	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)

commands/commandeer.go

@@ -27,7 +27,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"strings"
 	"sync"
 	"time"
 
@@ -388,21 +387,23 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 
 	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())
 
-	themeDir := c.hugo.PathSpec.GetFirstThemeDir()
-	if themeDir != "" {
-		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
-			return newSystemError("Unable to find theme Directory:", themeDir)
+	// TODO(bep) mod
+	/*
+		themeDir := c.hugo.PathSpec.GetFirstThemeDir()
+		if themeDir != "" {
+			if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
+				return newSystemError("Unable to find theme Directory:", themeDir)
+			}
 		}
-	}
 
-	dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)
-
-	if themeVersionMismatch {
-		name := filepath.Base(dir)
-		cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
-			strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)
-	}
+		dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)
 
+		if themeVersionMismatch {
+			name := filepath.Base(dir)
+			cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
+				strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)
+		}
+	*/
 	return nil
 
 }

commands/commands.go

@@ -54,6 +54,7 @@ func (b *commandsBuilder) addAll() *commandsBuilder {
 		newImportCmd(),
 		newGenCmd(),
 		createReleaser(),
+		b.newModCmd(),
 	)
 
 	return b
@@ -189,9 +190,10 @@ Complete documentation is available at http://gohugo.io/.`,
 }
 
 type hugoBuilderCommon struct {
-	source      string
-	baseURL     string
-	environment string
+	source       string
+	baseURL      string
+	environment  string
+	ignoreVendor bool
 
 	buildWatch bool
 
@@ -243,20 +245,26 @@ func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string {
 	return hugo.EnvironmentProduction
 }
 
+func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) {
+	cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+	cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment")
+	cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory")
+	cmd.PersistentFlags().BoolP("ignoreVendor", "", false, "ignores any _vendor directory")
+}
+
 func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
+	cc.handleCommonBuilderFlags(cmd)
 	cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories")
 	cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft")
 	cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future")
 	cmd.Flags().BoolP("buildExpired", "E", false, "include expired content")
-	cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
-	cmd.Flags().StringVarP(&cc.environment, "environment", "e", "", "build environment")
 	cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")
 	cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory")
 	cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
 	cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory")
 	cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to")
 	cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)")
-	cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory")
 	cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/")
 	cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages")
 	cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build")

commands/convert.go

@@ -30,7 +30,6 @@ import (
 	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/gohugoio/hugo/parser/pageparser"
 
-	src "github.com/gohugoio/hugo/source"
 	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/hugolib"
@@ -152,8 +151,8 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target
 
 	site.Log.INFO.Println("Attempting to convert", p.File().Filename())
 
-	f, _ := p.File().(src.ReadableFile)
-	file, err := f.Open()
+	f := p.File()
+	file, err := f.FileInfo().Meta().Open()
 	if err != nil {
 		site.Log.ERROR.Println(errMsg)
 		file.Close()