Add SCSS support and other related Hugo Pipes

Fixes gohugoio#4381

.travis.yml

@@ -18,8 +18,9 @@ install:
   - go get github.com/magefile/mage
   - mage -v vendor
 script:
-  - mage -v hugoRace
+  - mage -v test
   - mage -v check
+  - mage -v hugo
   - ./hugo -s docs/
   - ./hugo --renderToMemory -s docs/
 before_install:

Gopkg.toml

@@ -16,6 +16,14 @@
   branch = "master"
   name = "github.com/bep/gitmap"
 
+[[constraint]]
+   branch = "master"
+   name = "github.com/bep/go-tocss"
+
+[[override]]
+   branch = "master"
+   name = "github.com/wellington/go-libsass"
+
 [[constraint]]
  name = "github.com/chaseadamsio/goorgeous"
  version = "^1.1.0"
@@ -149,3 +157,15 @@
 [[constraint]]
   name = "github.com/bep/debounce"
   version = "^1.1.0"
+
+[[constraint]]
+  name = "github.com/tdewolff/minify"
+  version = "^2.3.5"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/BurntSushi/locker"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/hashstructure"

commands/commandeer.go

@@ -152,6 +152,8 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 		doWithCommandeer,
 		doWithConfig)
 
+	config.Set("isServer", running)
+
 	if err != nil {
 		if mustHaveConfigFile {
 			return err

commands/hugo.go

@@ -552,8 +552,8 @@ func (c *commandeer) getDirList() ([]string, error) {
 	// SymbolicWalk will log anny ERRORs
 	// Also note that the Dirnames fetched below will contain any relevant theme
 	// directories.
-	for _, contentDir := range c.hugo.PathSpec.BaseFs.AbsContentDirs {
-		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker)
+	for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames {
+		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker)
 	}
 
 	for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames {
@@ -574,6 +574,10 @@ func (c *commandeer) getDirList() ([]string, error) {
 		}
 	}
 
+	for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames {
+		_ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker)
+	}
+
 	if len(nested) > 0 {
 		for {
 

common/errors/errors.go

@@ -0,0 +1,23 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errors contains common Hugo errors and error related utilities.
+package errors
+
+import (
+	"errors"
+)
+
+// We will, at least to begin with, make some Hugo features (SCSS with libsass) optional,
+// and this error is used to signal those situations.
+var FeatureNotAvailableErr = errors.New("this feature is not available in your current Hugo version")

create/content_test.go

@@ -88,6 +88,8 @@ func initViper(v *viper.Viper) {
 	v.Set("i18nDir", "i18n")
 	v.Set("theme", "sample")
 	v.Set("archetypeDir", "archetypes")
+	v.Set("resourceDir", "resources")
+	v.Set("publishDir", "public")
 }
 
 func initFs(fs *hugofs.Fs) error {
@@ -191,6 +193,7 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) {
 	v.Set("i18nDir", "i18n")
 	v.Set("layoutDir", "layouts")
 	v.Set("archetypeDir", "archetypes")
+	v.Set("assetDir", "assets")
 
 	fs := hugofs.NewMem(v)
 

deps/deps.go

@@ -1,17 +1,18 @@
 package deps
 
 import (
-	"io/ioutil"
-	"log"
-	"os"
 	"time"
 
+	"github.com/gohugoio/hugo/common/loggers"
+
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/metrics"
 	"github.com/gohugoio/hugo/output"
+	"github.com/gohugoio/hugo/resource"
 	"github.com/gohugoio/hugo/source"
 	"github.com/gohugoio/hugo/tpl"
 	jww "github.com/spf13/jwalterweatherman"
@@ -42,6 +43,9 @@ type Deps struct {
 	// The SourceSpec to use
 	SourceSpec *source.SourceSpec `json:"-"`
 
+	// The Resource Spec to use
+	ResourceSpec *resource.Spec
+
 	// The configuration to use
 	Cfg config.Provider `json:"-"`
 
@@ -115,7 +119,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 	}
 
 	if logger == nil {
-		logger = jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+		logger = loggers.NewErrorLogger()
 	}
 
 	if fs == nil {
@@ -129,6 +133,11 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
+	resourceSpec, err := resource.NewSpec(ps, logger, cfg.MediaTypes)
+	if err != nil {
+		return nil, err
+	}
+
 	contentSpec, err := helpers.NewContentSpec(cfg.Language)
 	if err != nil {
 		return nil, err
@@ -153,6 +162,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		PathSpec:            ps,
 		ContentSpec:         contentSpec,
 		SourceSpec:          sp,
+		ResourceSpec:        resourceSpec,
 		Cfg:                 cfg.Language,
 		Language:            cfg.Language,
 		Timeout:             time.Duration(timeoutms) * time.Millisecond,
@@ -167,7 +177,8 @@ func New(cfg DepsCfg) (*Deps, error) {
 
 // ForLanguage creates a copy of the Deps with the language dependent
 // parts switched out.
-func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) {
+func (d Deps) ForLanguage(cfg DepsCfg) (*Deps, error) {
+	l := cfg.Language
 	var err error
 
 	d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs)
@@ -180,6 +191,11 @@ func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) {
 		return nil, err
 	}
 
+	d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.Log, cfg.MediaTypes)
+	if err != nil {
+		return nil, err
+	}
+
 	d.Cfg = l
 	d.Language = l
 
@@ -212,6 +228,9 @@ type DepsCfg struct {
 	// The configuration to use.
 	Cfg config.Provider
 
+	// The media types configured.
+	MediaTypes media.Types
+
 	// Template handling.
 	TemplateProvider ResourceProvider
 	WithTemplate     func(templ tpl.TemplateHandler) error

helpers/general.go

@@ -356,7 +356,7 @@ func MD5String(f string) string {
 // MD5FromFileFast creates a MD5 hash from the given file. It only reads parts of
 // the file for speed, so don't use it if the files are very subtly different.
 // It will not close the file.
-func MD5FromFileFast(f afero.File) (string, error) {
+func MD5FromFileFast(r io.ReadSeeker) (string, error) {
 	const (
 		// Do not change once set in stone!
 		maxChunks = 8
@@ -369,7 +369,7 @@ func MD5FromFileFast(f afero.File) (string, error) {
 
 	for i := 0; i < maxChunks; i++ {
 		if i > 0 {
-			_, err := f.Seek(seek, 0)
+			_, err := r.Seek(seek, 0)
 			if err != nil {
 				if err == io.EOF {
 					break
@@ -378,7 +378,7 @@ func MD5FromFileFast(f afero.File) (string, error) {
 			}
 		}
 
-		_, err := io.ReadAtLeast(f, buff, peekSize)
+		_, err := io.ReadAtLeast(r, buff, peekSize)
 		if err != nil {
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				h.Write(buff)

helpers/path.go

@@ -222,12 +222,22 @@ func GetDottedRelativePath(inPath string) string {
 	return dottedPath
 }
 
+// ExtNoDelimiter takes a path and returns the extension, excluding the delmiter, i.e. "md".
+func ExtNoDelimiter(in string) string {
+	return strings.TrimPrefix(Ext(in), ".")
+}
+
 // Ext takes a path and returns the extension, including the delmiter, i.e. ".md".
 func Ext(in string) string {
 	_, ext := fileAndExt(in, fpb)
 	return ext
 }
 
+// PathAndExt is the same as FileAndExt, but it uses the path package.
+func PathAndExt(in string) (string, string) {
+	return fileAndExt(in, pb)
+}
+
 // FileAndExt takes a path and returns the file and extension separated,
 // the extension including the delmiter, i.e. ".md".
 func FileAndExt(in string) (string, string) {

helpers/path_test.go

@@ -78,6 +78,9 @@ func TestMakePathSanitized(t *testing.T) {
 	v.Set("dataDir", "data")
 	v.Set("i18nDir", "i18n")
 	v.Set("layoutDir", "layouts")
+	v.Set("assetDir", "assets")
+	v.Set("resourceDir", "resources")
+	v.Set("publishDir", "public")
 	v.Set("archetypeDir", "archetypes")
 
 	l := langs.NewDefaultLanguage(v)
@@ -475,6 +478,7 @@ func createTempDirWithNonZeroLengthFiles() (string, error) {
 		return "", fileErr
 	}
 	byteString := []byte("byteString")
+
 	fileErr = ioutil.WriteFile(f.Name(), byteString, 0644)
 	if fileErr != nil {
 		// delete the file
@@ -585,6 +589,11 @@ func TestAbsPathify(t *testing.T) {
 
 }
 
+func TestExtNoDelimiter(t *testing.T) {
+	assert := require.New(t)
+	assert.Equal("json", ExtNoDelimiter(filepath.FromSlash("/my/data.json")))
+}
+
 func TestFilename(t *testing.T) {
 	type test struct {
 		input, expected string