Skip to content
Permalink
Browse files

Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
  • Loading branch information...
bep committed May 3, 2019
1 parent 4795314 commit 9f5a92078a3f388b52d597b5a59af5c933a112d2
Showing with 9,809 additions and 5,347 deletions.
  1. +1 −2 benchbep.sh
  2. +27 −22 cache/filecache/filecache.go
  3. +42 −14 cache/filecache/filecache_config.go
  4. +17 −28 cache/filecache/filecache_config_test.go
  5. +82 −35 cache/filecache/filecache_pruner.go
  6. +2 −9 cache/filecache/filecache_pruner_test.go
  7. +53 −11 cache/filecache/filecache_test.go
  8. +10 −18 commands/commandeer.go
  9. +10 −3 commands/commands.go
  10. +2 −3 commands/convert.go
  11. +138 −176 commands/hugo.go
  12. +8 −63 commands/import_jekyll.go
  13. +1 −0 commands/list_test.go
  14. +189 −0 commands/mod.go
  15. +2 −4 commands/new.go
  16. +13 −8 commands/server.go
  17. +1 −0 commands/static_syncer.go
  18. +11 −0 common/herrors/errors.go
  19. +90 −0 common/hugio/copy.go
  20. +1 −1 common/hugo/vars_extended.go
  21. +1 −1 common/hugo/vars_regular.go
  22. +15 −1 common/hugo/version.go
  23. +6 −0 common/hugo/version_test.go
  24. +4 −1 common/loggers/loggers.go
  25. +96 −0 common/maps/params.go
  26. +45 −0 common/maps/params_test.go
  27. +0 −2 config/configLoader.go
  28. +6 −2 config/configProvider.go
  29. +24 −0 config/env.go
  30. +13 −20 hugofs/nolstat_fs.go → config/env_test.go
  31. +58 −22 create/content.go
  32. +4 −1 create/content_template_handler.go
  33. +51 −38 create/content_test.go
  34. +3 −3 deps/deps.go
  35. +9 −5 go.mod
  36. +92 −0 go.sum
  37. +0 −6 helpers/content.go
  38. +0 −10 helpers/content_test.go
  39. +1 −2 helpers/docshelper.go
  40. +45 −12 helpers/general.go
  41. +70 −0 helpers/general_test.go
  42. +114 −114 helpers/path.go
  43. +25 −40 helpers/path_test.go
  44. +7 −2 helpers/pathspec_test.go
  45. +9 −0 helpers/testhelpers_test.go
  46. +39 −0 htesting/test_helpers.go
  47. +0 −100 htesting/test_structs.go
  48. +0 −91 hugofs/basepath_real_filename_fs.go
  49. +205 −0 hugofs/decorators.go
  50. +297 −0 hugofs/fileinfo.go
  51. +121 −0 hugofs/files/classifier.go
  52. +49 −0 hugofs/files/classifier_test.go
  53. +341 −0 hugofs/filter_fs.go
  54. +48 −0 hugofs/filter_fs_test.go
  55. +4 −2 hugofs/fs.go
  56. +38 −2 hugofs/language_composite_fs.go
  57. +0 −107 hugofs/language_composite_fs_test.go
  58. +0 −346 hugofs/language_fs.go
  59. +0 −100 hugofs/language_fs_test.go
  60. +85 −0 hugofs/nosymlink_fs.go
  61. +97 −0 hugofs/nosymlink_test.go
  62. +364 −93 hugofs/rootmapping_fs.go
  63. +188 −11 hugofs/rootmapping_fs_test.go
  64. +293 −0 hugofs/slice_fs.go
  65. +308 −0 hugofs/walk.go
  66. +225 −0 hugofs/walk_test.go
  67. +150 −164 hugolib/config.go
  68. +143 −18 hugolib/config_test.go
  69. +1 −0 hugolib/data/hugo.toml
  70. +34 −55 hugolib/disableKinds_test.go
  71. +29 −47 hugolib/fileInfo.go
  72. +1 −0 hugolib/fileInfo_test.go
  73. +324 −380 hugolib/filesystems/basefs.go
  74. +121 −66 hugolib/filesystems/basefs_test.go
  75. +492 −0 hugolib/hugo_modules_test.go
  76. +110 −88 hugolib/hugo_sites.go
  77. +4 −4 hugolib/hugo_sites_build.go
  78. +2 −1 hugolib/hugo_sites_build_errors_test.go
  79. +14 −40 hugolib/hugo_sites_build_test.go
  80. +0 −268 hugolib/hugo_themes_test.go
  81. +8 −3 hugolib/language_content_dir_test.go
  82. +19 −25 hugolib/menu_test.go
  83. +0 −56 hugolib/multilingual.go
  84. +8 −10 hugolib/page.go
  85. +7 −6 hugolib/page__meta.go
  86. +2 −1 hugolib/page_permalink_test.go
  87. +29 −23 hugolib/page_test.go
  88. +0 −206 hugolib/pagebundler.go
  89. +0 −773 hugolib/pagebundler_capture.go
  90. +0 −272 hugolib/pagebundler_capture_test.go
  91. +0 −305 hugolib/pagebundler_handlers.go
  92. +244 −140 hugolib/pagebundler_test.go
  93. +0 −1 hugolib/pagecollections.go
  94. +779 −0 hugolib/pages_capture.go
  95. +88 −0 hugolib/pages_capture_test.go
  96. +34 −32 hugolib/paths/paths.go
  97. +8 −1 hugolib/paths/paths_test.go
  98. +0 −154 hugolib/paths/themes.go
  99. +35 −31 hugolib/resource_chain_test.go
  100. +32 −38 hugolib/shortcode_test.go
  101. +14 −90 hugolib/site.go
  102. +20 −27 hugolib/site_output_test.go
  103. +2 −1 hugolib/site_sections_test.go
  104. +10 −13 hugolib/site_stats_test.go
  105. +4 −1 hugolib/site_test.go
  106. +25 −29 hugolib/taxonomy_test.go
  107. +2 −1 hugolib/template_engines_test.go
  108. +13 −10 hugolib/template_test.go
  109. +201 −67 hugolib/testhelpers_test.go
  110. +217 −0 langs/config.go
  111. +11 −2 langs/i18n/i18n_test.go
  112. +32 −24 langs/i18n/translationProvider.go
  113. +19 −6 langs/language.go
  114. +18 −3 magefile.go
  115. +570 −0 modules/client.go
  116. +117 −0 modules/client_test.go
  117. +574 −0 modules/collect.go
  118. +38 −0 modules/collect_test.go
  119. +335 −0 modules/config.go
  120. +132 −0 modules/config_test.go
  121. +196 −0 modules/module.go
  122. +1 −1 output/layout.go
  123. +26 −0 parser/metadecoders/decoder.go
  124. +32 −0 parser/metadecoders/decoder_test.go
  125. +1 −1 resources/image_cache.go
  126. +3 −2 resources/page/page_nop.go
  127. +2 −2 resources/page/page_wrappers.autogen.go
  128. +1 −1 resources/page/pagemeta/page_frontmatter.go
  129. +5 −0 resources/page/permalinks.go
  130. +71 −0 resources/page/site.go
  131. +9 −2 resources/page/testhelpers_test.go
  132. +2 −2 resources/page/zero_file.autogen.go
  133. +19 −25 resources/resource.go
  134. +2 −61 resources/resource/params.go
  135. +2 −2 resources/resource_factories/bundler/bundler.go
  136. +6 −6 resources/resource_factories/create/create.go
  137. +9 −3 resources/resource_test.go
  138. +2 −2 resources/resource_transformers/postcss/postcss.go
  139. +2 −1 resources/resource_transformers/tocss/scss/client.go
  140. +8 −3 resources/resource_transformers/tocss/scss/tocss.go
  141. +23 −36 resources/testhelpers_test.go
  142. +4 −0 resources/transform.go
  143. +43 −39 source/fileInfo.go
  144. +4 −53 source/fileInfo_test.go
  145. +45 −51 source/filesystem.go
  146. +36 −9 source/filesystem_test.go
  147. +13 −0 source/sourceSpec.go
  148. +8 −2 tpl/cast/docshelper.go
  149. +2 −0 tpl/data/init_test.go
  150. +12 −5 tpl/data/resources_test.go
  151. +5 −3 tpl/hugo/init_test.go
  152. +15 −10 tpl/os/os.go
  153. +6 −2 tpl/site/init_test.go
  154. +5 −2 tpl/template.go
  155. +1 −2 tpl/tplimpl/embedded/templates.autogen.go
  156. +3 −3 tpl/tplimpl/template.go
  157. +12 −2 tpl/tplimpl/template_funcs_test.go
  158. +1 −1 transform/livereloadinject/livereloadinject_test.go
@@ -1,2 +1 @@
gobench -package=./hugolib -bench="BenchmarkSiteBuilding/YAML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render" -count=3 > 1.bench
benchcmp -best 0.bench 1.bench
gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"
@@ -44,6 +44,9 @@ type Cache struct {
// 0 is effectively turning this cache off.
maxAge time.Duration

// When set, we just remove this entire root directory on expiration.
pruneAllRootDir string

nlocker *lockTracker
}

@@ -77,11 +80,12 @@ type ItemInfo struct {
}

// NewCache creates a new file cache with the given filesystem and max age.
func NewCache(fs afero.Fs, maxAge time.Duration) *Cache {
func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
return &Cache{
Fs: fs,
nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
maxAge: maxAge,
Fs: fs,
nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
maxAge: maxAge,
pruneAllRootDir: pruneAllRootDir,
}
}

@@ -307,9 +311,15 @@ func (f Caches) Get(name string) *Cache {
// NewCaches creates a new set of file caches from the given
// configuration.
func NewCaches(p *helpers.PathSpec) (Caches, error) {
dcfg, err := decodeConfig(p)
if err != nil {
return nil, err
var dcfg Configs
if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {
dcfg = c
} else {
var err error
dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)
if err != nil {
return nil, err
}
}

fs := p.Fs.Source
@@ -319,30 +329,25 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
var cfs afero.Fs

if v.isResourceDir {
cfs = p.BaseFs.Resources.Fs
cfs = p.BaseFs.ResourcesCache
} else {
cfs = fs
}

var baseDir string
if !strings.HasPrefix(v.Dir, "_gen") {
// We do cache eviction (file removes) and since the user can set
// his/hers own cache directory, we really want to make sure
// we do not delete any files that do not belong to this cache.
// We do add the cache name as the root, but this is an extra safe
// guard. We skip the files inside /resources/_gen/ because
// that would be breaking.
baseDir = filepath.Join(v.Dir, filecacheRootDirname, k)
} else {
baseDir = filepath.Join(v.Dir, k)
}
if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
baseDir := v.Dir

if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
return nil, err
}

bfs := afero.NewBasePathFs(cfs, baseDir)

m[k] = NewCache(bfs, v.MaxAge)
var pruneAllRootDir string
if k == cacheKeyModules {
pruneAllRootDir = "pkg"
}

m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
}

return m, nil
@@ -19,6 +19,8 @@ import (
"strings"
"time"

"github.com/gohugoio/hugo/config"

"github.com/gohugoio/hugo/helpers"

"github.com/mitchellh/mapstructure"
@@ -32,7 +34,7 @@ const (
resourcesGenDir = ":resourceDir/_gen"
)

var defaultCacheConfig = cacheConfig{
var defaultCacheConfig = Config{
MaxAge: -1, // Never expire
Dir: ":cacheDir/:project",
}
@@ -42,9 +44,20 @@ const (
cacheKeyGetCSV = "getcsv"
cacheKeyImages = "images"
cacheKeyAssets = "assets"
cacheKeyModules = "modules"
)

var defaultCacheConfigs = map[string]cacheConfig{
type Configs map[string]Config

func (c Configs) CacheDirModules() string {
return c[cacheKeyModules].Dir
}

var defaultCacheConfigs = Configs{
cacheKeyModules: {
MaxAge: -1,
Dir: ":cacheDir/modules",
},
cacheKeyGetJSON: defaultCacheConfig,
cacheKeyGetCSV: defaultCacheConfig,
cacheKeyImages: {
@@ -57,9 +70,7 @@ var defaultCacheConfigs = map[string]cacheConfig{
},
}

type cachesConfig map[string]cacheConfig

type cacheConfig struct {
type Config struct {
// Max age of cache entries in this cache. Any items older than this will
// be removed and not returned from the cache.
// a negative value means forever, 0 means cache is disabled.
@@ -88,25 +99,28 @@ func (f Caches) ImageCache() *Cache {
return f[cacheKeyImages]
}

// ModulesCache gets the file cache for Hugo Modules.
func (f Caches) ModulesCache() *Cache {
return f[cacheKeyModules]
}

// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
func (f Caches) AssetsCache() *Cache {
return f[cacheKeyAssets]
}

func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
c := make(cachesConfig)
func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) {
c := make(Configs)
valid := make(map[string]bool)
// Add defaults
for k, v := range defaultCacheConfigs {
c[k] = v
valid[k] = true
}

cfg := p.Cfg

m := cfg.GetStringMap(cachesConfigKey)

_, isOsFs := p.Fs.Source.(*afero.OsFs)
_, isOsFs := fs.(*afero.OsFs)

for k, v := range m {
cc := defaultCacheConfig
@@ -148,7 +162,7 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {

for i, part := range parts {
if strings.HasPrefix(part, ":") {
resolved, isResource, err := resolveDirPlaceholder(p, part)
resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part)
if err != nil {
return c, err
}
@@ -176,6 +190,18 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
}
}

if !strings.HasPrefix(v.Dir, "_gen") {
// We do cache eviction (file removes) and since the user can set
// his/hers own cache directory, we really want to make sure
// we do not delete any files that do not belong to this cache.
// We do add the cache name as the root, but this is an extra safe
// guard. We skip the files inside /resources/_gen/ because
// that would be breaking.
v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k)
} else {
v.Dir = filepath.Join(v.Dir, k)
}

if disabled {
v.MaxAge = 0
}
@@ -187,15 +213,17 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
}

// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {
func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) {
workingDir := cfg.GetString("workingDir")

switch strings.ToLower(placeholder) {
case ":resourcedir":
return "", true, nil
case ":cachedir":
d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)
d, err := helpers.GetCacheDir(fs, cfg)
return d, false, err
case ":project":
return filepath.Base(p.WorkingDir), false, nil
return filepath.Base(workingDir), false, nil
}

return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
@@ -20,10 +20,9 @@ import (
"testing"
"time"

"github.com/gohugoio/hugo/helpers"
"github.com/spf13/afero"

"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"

"github.com/spf13/viper"
"github.com/stretchr/testify/require"
@@ -57,22 +56,19 @@ dir = "/path/to/c3"

cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
fs := hugofs.NewMem(cfg)
p, err := helpers.NewPathSpec(fs, cfg)
fs := afero.NewMemMapFs()
decoded, err := DecodeConfig(fs, cfg)
assert.NoError(err)

decoded, err := decodeConfig(p)
assert.NoError(err)

assert.Equal(4, len(decoded))
assert.Equal(5, len(decoded))

c2 := decoded["getcsv"]
assert.Equal("11h0m0s", c2.MaxAge.String())
assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
assert.Equal(filepath.FromSlash("/path/to/c2/filecache/getcsv"), c2.Dir)

c3 := decoded["images"]
assert.Equal(time.Duration(-1), c3.MaxAge)
assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
assert.Equal(filepath.FromSlash("/path/to/c3/filecache/images"), c3.Dir)

}

@@ -105,14 +101,11 @@ dir = "/path/to/c3"

cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
fs := hugofs.NewMem(cfg)
p, err := helpers.NewPathSpec(fs, cfg)
assert.NoError(err)

decoded, err := decodeConfig(p)
fs := afero.NewMemMapFs()
decoded, err := DecodeConfig(fs, cfg)
assert.NoError(err)

assert.Equal(4, len(decoded))
assert.Equal(5, len(decoded))

for _, v := range decoded {
assert.Equal(time.Duration(0), v.MaxAge)
@@ -133,24 +126,22 @@ func TestDecodeConfigDefault(t *testing.T) {
cfg.Set("cacheDir", "/cache/thecache")
}

fs := hugofs.NewMem(cfg)
p, err := helpers.NewPathSpec(fs, cfg)
assert.NoError(err)
fs := afero.NewMemMapFs()

decoded, err := decodeConfig(p)
decoded, err := DecodeConfig(fs, cfg)

assert.NoError(err)

assert.Equal(4, len(decoded))
assert.Equal(5, len(decoded))

imgConfig := decoded[cacheKeyImages]
jsonConfig := decoded[cacheKeyGetJSON]

if runtime.GOOS == "windows" {
assert.Equal("_gen", imgConfig.Dir)
assert.Equal(filepath.FromSlash("_gen/images"), imgConfig.Dir)
} else {
assert.Equal("_gen", imgConfig.Dir)
assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir)
assert.Equal("_gen/images", imgConfig.Dir)
assert.Equal("/cache/thecache/hugoproject/filecache/getjson", jsonConfig.Dir)
}

assert.True(imgConfig.isResourceDir)
@@ -183,11 +174,9 @@ dir = "/"

cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
fs := hugofs.NewMem(cfg)
p, err := helpers.NewPathSpec(fs, cfg)
assert.NoError(err)
fs := afero.NewMemMapFs()

_, err = decodeConfig(p)
_, err = DecodeConfig(fs, cfg)
assert.Error(err)

}

0 comments on commit 9f5a920

Please sign in to comment.
You can’t perform that action at this time.