Add readFile template func

This also includes a refactor of the hugofs package and its usage.

The motivation for that is:

The Afero filesystems are brilliant. Hugo's way of adding a dozen of global variables for the different filesystems was a mistake. In readFile (and also in some other places in Hugo today) we need a way to restrict the access inside the working dir. We could use ioutil.ReadFile and implement the path checking, checking the base path and the dots ("..") etc. But it is obviously better to use an Afero BasePathFs combined witha ReadOnlyFs. We could create a use-once-filesystem and handle the initialization ourselves, but since this is also useful to others and the initialization depends on some other global state (which would mean to create a new file system on every invocation), we might as well do it properly and encapsulate the predefined set of filesystems. This change also leads the way, if needed, to encapsulate the file systems in a struct, making it possible to have several file system sets in action at once (parallel multilanguage site building? With Moore's law and all...)

Fixes gohugoio#1551

commands/gendoc.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -51,9 +51,9 @@ for rendering in Hugo.`,
 		if !strings.HasSuffix(gendocdir, helpers.FilePathSeparator) {
 			gendocdir += helpers.FilePathSeparator
 		}
-		if found, _ := helpers.Exists(gendocdir, hugofs.OsFs); !found {
+		if found, _ := helpers.Exists(gendocdir, hugofs.Os()); !found {
 			jww.FEEDBACK.Println("Directory", gendocdir, "does not exist, creating...")
-			hugofs.OsFs.MkdirAll(gendocdir, 0777)
+			hugofs.Os().MkdirAll(gendocdir, 0777)
 		}
 		now := time.Now().Format(time.RFC3339)
 		prepender := func(filename string) string {

commands/genman.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -41,9 +41,9 @@ in the "man" directory under the current directory.`,
 		if !strings.HasSuffix(genmandir, helpers.FilePathSeparator) {
 			genmandir += helpers.FilePathSeparator
 		}
-		if found, _ := helpers.Exists(genmandir, hugofs.OsFs); !found {
+		if found, _ := helpers.Exists(genmandir, hugofs.Os()); !found {
 			jww.FEEDBACK.Println("Directory", genmandir, "does not exist, creating...")
-			hugofs.OsFs.MkdirAll(genmandir, 0777)
+			hugofs.Os().MkdirAll(genmandir, 0777)
 		}
 		cmd.Root().DisableAutoGenTag = true
 

commands/hugo.go

@@ -423,14 +423,14 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
 		if helpers.FilePathSeparator != cacheDir[len(cacheDir)-1:] {
 			cacheDir = cacheDir + helpers.FilePathSeparator
 		}
-		isDir, err := helpers.DirExists(cacheDir, hugofs.SourceFs)
+		isDir, err := helpers.DirExists(cacheDir, hugofs.Source())
 		utils.CheckErr(err)
 		if isDir == false {
 			mkdir(cacheDir)
 		}
 		viper.Set("CacheDir", cacheDir)
 	} else {
-		viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.SourceFs))
+		viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.Source()))
 	}
 
 	if verboseLog || logging || (viper.IsSet("LogFile") && viper.GetString("LogFile") != "") {
@@ -453,6 +453,9 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
 
 	jww.INFO.Println("Using config file:", viper.ConfigFileUsed())
 
+	// Init file systems. This may be changed at a later point.
+	hugofs.InitDefaultFs()
+
 	themeDir := helpers.GetThemeDir()
 	if themeDir != "" {
 		if _, err := os.Stat(themeDir); os.IsNotExist(err) {
@@ -498,7 +501,7 @@ func build(watches ...bool) error {
 	// This is only used for benchmark testing. Cause the content is only visible
 	// in memory
 	if renderToMemory {
-		hugofs.DestinationFS = new(afero.MemMapFs)
+		hugofs.SetDestination(new(afero.MemMapFs))
 		// Rendering to memoryFS, publish to Root regardless of publishDir.
 		viper.Set("PublishDir", "/")
 	}
@@ -524,7 +527,7 @@ func build(watches ...bool) error {
 }
 
 func getStaticSourceFs() afero.Fs {
-	source := hugofs.SourceFs
+	source := hugofs.Source()
 	themeDir, err := helpers.GetThemeStaticDirPath()
 	staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator
 
@@ -563,8 +566,8 @@ func getStaticSourceFs() afero.Fs {
 	jww.INFO.Println("using a UnionFS for static directory comprised of:")
 	jww.INFO.Println("Base:", themeDir)
 	jww.INFO.Println("Overlay:", staticDir)
-	base := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, themeDir))
-	overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, staticDir))
+	base := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.Source(), themeDir))
+	overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.Source(), staticDir))
 	return afero.NewCopyOnWriteFs(base, overlay)
 }
 
@@ -587,7 +590,7 @@ func copyStatic() error {
 	syncer := fsync.NewSyncer()
 	syncer.NoTimes = viper.GetBool("notimes")
 	syncer.SrcFs = staticSourceFs
-	syncer.DestFs = hugofs.DestinationFS
+	syncer.DestFs = hugofs.Destination()
 	// Now that we are using a unionFs for the static directories
 	// We can effectively clean the publishDir on initial sync
 	syncer.Delete = viper.GetBool("cleanDestinationDir")
@@ -653,12 +656,12 @@ func getDirList() []string {
 		return nil
 	}
 
-	helpers.SymbolicWalk(hugofs.SourceFs, dataDir, walker)
-	helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("ContentDir")), walker)
-	helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("LayoutDir")), walker)
-	helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("StaticDir")), walker)
+	helpers.SymbolicWalk(hugofs.Source(), dataDir, walker)
+	helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("ContentDir")), walker)
+	helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("LayoutDir")), walker)
+	helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("StaticDir")), walker)
 	if helpers.ThemeSet() {
-		helpers.SymbolicWalk(hugofs.SourceFs, helpers.AbsPathify(viper.GetString("themesDir")+"/"+viper.GetString("theme")), walker)
+		helpers.SymbolicWalk(hugofs.Source(), helpers.AbsPathify(viper.GetString("themesDir")+"/"+viper.GetString("theme")), walker)
 	}
 
 	return a
@@ -770,8 +773,8 @@ func NewWatcher(port int) error {
 					// recursively add new directories to watch list
 					// When mkdir -p is used, only the top directory triggers an event (at least on OSX)
 					if ev.Op&fsnotify.Create == fsnotify.Create {
-						if s, err := hugofs.SourceFs.Stat(ev.Name); err == nil && s.Mode().IsDir() {
-							helpers.SymbolicWalk(hugofs.SourceFs, ev.Name, walkAdder)
+						if s, err := hugofs.Source().Stat(ev.Name); err == nil && s.Mode().IsDir() {
+							helpers.SymbolicWalk(hugofs.Source(), ev.Name, walkAdder)
 						}
 					}
 
@@ -813,7 +816,7 @@ func NewWatcher(port int) error {
 						syncer := fsync.NewSyncer()
 						syncer.NoTimes = viper.GetBool("notimes")
 						syncer.SrcFs = staticSourceFs
-						syncer.DestFs = hugofs.DestinationFS
+						syncer.DestFs = hugofs.Destination()
 
 						// prevent spamming the log on changes
 						logger := helpers.NewDistinctFeedbackLogger()
@@ -858,7 +861,7 @@ func NewWatcher(port int) error {
 									// If file doesn't exist in any static dir, remove it
 									toRemove := filepath.Join(publishDir, relPath)
 									logger.Println("File no longer exists in static dir, removing", toRemove)
-									hugofs.DestinationFS.RemoveAll(toRemove)
+									hugofs.Destination().RemoveAll(toRemove)
 								} else if err == nil {
 									// If file still exists, sync it
 									logger.Println("Syncing", relPath, "to", publishDir)
@@ -939,7 +942,7 @@ func isThemeVsHugoVersionMismatch() (mismatch bool, requiredMinVersion string) {
 
 	themeDir := helpers.GetThemeDir()
 
-	fs := hugofs.SourceFs
+	fs := hugofs.Source()
 	path := filepath.Join(themeDir, "theme.toml")
 
 	exists, err := helpers.Exists(path, fs)

commands/import_jekyll.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -124,7 +124,7 @@ func importFromJekyll(cmd *cobra.Command, args []string) error {
 		return convertJekyllPost(path, relPath, targetDir, draft)
 	}
 
-	err = helpers.SymbolicWalk(hugofs.OsFs, jekyllRoot, callback)
+	err = helpers.SymbolicWalk(hugofs.Os(), jekyllRoot, callback)
 
 	if err != nil {
 		return err
@@ -139,7 +139,7 @@ func importFromJekyll(cmd *cobra.Command, args []string) error {
 
 // TODO: Consider calling doNewSite() instead?
 func createSiteFromJekyll(jekyllRoot, targetDir string, force bool) error {
-	fs := hugofs.SourceFs
+	fs := hugofs.Source()
 	if exists, _ := helpers.Exists(targetDir, fs); exists {
 		if isDir, _ := helpers.IsDir(targetDir, fs); !isDir {
 			return errors.New("Target path \"" + targetDir + "\" already exists but not a directory")
@@ -187,7 +187,7 @@ func createSiteFromJekyll(jekyllRoot, targetDir string, force bool) error {
 }
 
 func loadJekyllConfig(jekyllRoot string) map[string]interface{} {
-	fs := hugofs.SourceFs
+	fs := hugofs.Source()
 	path := filepath.Join(jekyllRoot, "_config.yml")
 
 	exists, err := helpers.Exists(path, fs)
@@ -252,7 +252,7 @@ func createConfigFromJekyll(inpath string, kind string, jekyllConfig map[string]
 		return err
 	}
 
-	err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs)
+	err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.Source())
 	if err != nil {
 		return
 	}

commands/new.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -110,7 +110,7 @@ func NewContent(cmd *cobra.Command, args []string) error {
 		kind = contentType
 	}
 
-	return create.NewContent(hugofs.SourceFs, kind, createpath)
+	return create.NewContent(hugofs.Source(), kind, createpath)
 }
 
 func doNewSite(basepath string, force bool) error {
@@ -123,12 +123,12 @@ func doNewSite(basepath string, force bool) error {
 		filepath.Join(basepath, "themes"),
 	}
 
-	if exists, _ := helpers.Exists(basepath, hugofs.SourceFs); exists {
-		if isDir, _ := helpers.IsDir(basepath, hugofs.SourceFs); !isDir {
+	if exists, _ := helpers.Exists(basepath, hugofs.Source()); exists {
+		if isDir, _ := helpers.IsDir(basepath, hugofs.Source()); !isDir {
 			return errors.New(basepath + " already exists but not a directory")
 		}
 
-		isEmpty, _ := helpers.IsEmpty(basepath, hugofs.SourceFs)
+		isEmpty, _ := helpers.IsEmpty(basepath, hugofs.Source())
 
 		switch {
 		case !isEmpty && !force:
@@ -137,15 +137,15 @@ func doNewSite(basepath string, force bool) error {
 		case !isEmpty && force:
 			all := append(dirs, filepath.Join(basepath, "config."+configFormat))
 			for _, path := range all {
-				if exists, _ := helpers.Exists(path, hugofs.SourceFs); exists {
+				if exists, _ := helpers.Exists(path, hugofs.Source()); exists {
 					return errors.New(path + " already exists")
 				}
 			}
 		}
 	}
 
 	for _, dir := range dirs {
-		hugofs.SourceFs.MkdirAll(dir, 0777)
+		hugofs.Source().MkdirAll(dir, 0777)
 	}
 
 	createConfig(basepath, configFormat)
@@ -185,7 +185,7 @@ func NewTheme(cmd *cobra.Command, args []string) error {
 	createpath := helpers.AbsPathify(filepath.Join(viper.GetString("themesDir"), args[0]))
 	jww.INFO.Println("creating theme at", createpath)
 
-	if x, _ := helpers.Exists(createpath, hugofs.SourceFs); x {
+	if x, _ := helpers.Exists(createpath, hugofs.Source()); x {
 		return newUserError(createpath, "already exists")
 	}
 
@@ -204,7 +204,7 @@ func NewTheme(cmd *cobra.Command, args []string) error {
 
 	archDefault := []byte("+++\n+++\n")
 
-	err := helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), hugofs.SourceFs)
+	err := helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), hugofs.Source())
 	if err != nil {
 		return err
 	}
@@ -234,7 +234,7 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 `)
 
-	err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE.md"), bytes.NewReader(by), hugofs.SourceFs)
+	err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE.md"), bytes.NewReader(by), hugofs.Source())
 	if err != nil {
 		return err
 	}
@@ -256,7 +256,7 @@ func mkdir(x ...string) {
 func touchFile(x ...string) {
 	inpath := filepath.Join(x...)
 	mkdir(filepath.Dir(inpath))
-	err := helpers.WriteToDisk(inpath, bytes.NewReader([]byte{}), hugofs.SourceFs)
+	err := helpers.WriteToDisk(inpath, bytes.NewReader([]byte{}), hugofs.Source())
 	if err != nil {
 		jww.FATAL.Fatalln(err)
 	}
@@ -287,7 +287,7 @@ min_version = 0.15
   repo = ""
 `)
 
-	err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), hugofs.SourceFs)
+	err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), hugofs.Source())
 	if err != nil {
 		return
 	}
@@ -321,7 +321,7 @@ func createConfig(inpath string, kind string) (err error) {
 		return err
 	}
 
-	err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs)
+	err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.Source())
 	if err != nil {
 		return
 	}

commands/new_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2016 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/spf13/afero"
 	"github.com/spf13/hugo/hugofs"
 	"github.com/stretchr/testify/assert"
 )
@@ -41,14 +40,14 @@ func checkNewSiteInited(basepath string, t *testing.T) {
 	}
 
 	for _, path := range paths {
-		_, err := hugofs.SourceFs.Stat(path)
+		_, err := hugofs.Source().Stat(path)
 		assert.Nil(t, err)
 	}
 }
 
 func TestDoNewSite(t *testing.T) {
 	basepath := filepath.Join(os.TempDir(), "blog")
-	hugofs.SourceFs = new(afero.MemMapFs)
+	hugofs.InitMemFs()
 	err := doNewSite(basepath, false)
 	assert.Nil(t, err)
 
@@ -57,26 +56,26 @@ func TestDoNewSite(t *testing.T) {
 
 func TestDoNewSite_noerror_base_exists_but_empty(t *testing.T) {
 	basepath := filepath.Join(os.TempDir(), "blog")
-	hugofs.SourceFs = new(afero.MemMapFs)
-	hugofs.SourceFs.MkdirAll(basepath, 777)
+	hugofs.InitMemFs()
+	hugofs.Source().MkdirAll(basepath, 777)
 	err := doNewSite(basepath, false)
 	assert.Nil(t, err)
 }
 
 func TestDoNewSite_error_base_exists(t *testing.T) {
 	basepath := filepath.Join(os.TempDir(), "blog")
-	hugofs.SourceFs = new(afero.MemMapFs)
-	hugofs.SourceFs.MkdirAll(basepath, 777)
-	hugofs.SourceFs.Create(filepath.Join(basepath, "foo"))
+	hugofs.InitMemFs()
+	hugofs.Source().MkdirAll(basepath, 777)
+	hugofs.Source().Create(filepath.Join(basepath, "foo"))
 	// Since the directory already exists and isn't empty, expect an error
 	err := doNewSite(basepath, false)
 	assert.NotNil(t, err)
 }
 
 func TestDoNewSite_force_empty_dir(t *testing.T) {
 	basepath := filepath.Join(os.TempDir(), "blog")
-	hugofs.SourceFs = new(afero.MemMapFs)
-	hugofs.SourceFs.MkdirAll(basepath, 777)
+	hugofs.InitMemFs()
+	hugofs.Source().MkdirAll(basepath, 777)
 	err := doNewSite(basepath, true)
 	assert.Nil(t, err)
 
@@ -86,18 +85,18 @@ func TestDoNewSite_force_empty_dir(t *testing.T) {
 func TestDoNewSite_error_force_dir_inside_exists(t *testing.T) {
 	basepath := filepath.Join(os.TempDir(), "blog")
 	contentPath := filepath.Join(basepath, "content")
-	hugofs.SourceFs = new(afero.MemMapFs)
-	hugofs.SourceFs.MkdirAll(contentPath, 777)
+	hugofs.InitMemFs()
+	hugofs.Source().MkdirAll(contentPath, 777)
 	err := doNewSite(basepath, true)
 	assert.NotNil(t, err)
 }
 
 func TestDoNewSite_error_force_config_inside_exists(t *testing.T) {
 	basepath := filepath.Join(os.TempDir(), "blog")
 	configPath := filepath.Join(basepath, "config.toml")
-	hugofs.SourceFs = new(afero.MemMapFs)
-	hugofs.SourceFs.MkdirAll(basepath, 777)
-	hugofs.SourceFs.Create(configPath)
+	hugofs.InitMemFs()
+	hugofs.Source().MkdirAll(basepath, 777)
+	hugofs.Source().Create(configPath)
 	err := doNewSite(basepath, true)
 	assert.NotNil(t, err)
 }