diff --git a/.travis.yml b/.travis.yml index b04528203a7..e237e9b9099 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,8 +18,9 @@ install: - go get github.com/magefile/mage - mage -v vendor script: - - mage -v hugoRace + - mage -v test - mage -v check + - mage -v hugo - ./hugo -s docs/ - ./hugo --renderToMemory -s docs/ before_install: diff --git a/Gopkg.lock b/Gopkg.lock index 51fb96c52df..30cb215741a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,12 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + branch = "master" + name = "github.com/BurntSushi/locker" + packages = ["."] + revision = "a6e239ea1c69bff1cfdb20c4b73dadf52f784b6a" + [[projects]] branch = "master" name = "github.com/BurntSushi/toml" @@ -68,6 +74,16 @@ packages = ["."] revision = "012701e8669671499fc43e9792335a1dcbfe2afb" +[[projects]] + branch = "master" + name = "github.com/bep/go-tocss" + packages = [ + "scss", + "scss/libsass", + "tocss" + ] + revision = "471c87bebff471f8985f21b1290ac4520dd396c3" + [[projects]] name = "github.com/chaseadamsio/goorgeous" packages = ["."] @@ -107,6 +123,12 @@ revision = "487489b64fb796de2e55f4e8a4ad1e145f80e957" version = "v1.1.6" +[[projects]] + branch = "master" + name = "github.com/dsnet/golib" + packages = ["memfile"] + revision = "1ea1667757804fdcccc5a1810e09aba618885ac2" + [[projects]] branch = "master" name = "github.com/eknkc/amber" @@ -231,6 +253,12 @@ revision = "fd2f6c1403b37925bd7fe13af05853b8ae58ee5f" version = "v1.3.6" +[[projects]] + branch = "master" + name = "github.com/mitchellh/hashstructure" + packages = ["."] + revision = "2bca23e0e452137f789efbc8610126fd8b94f73b" + [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" @@ -355,6 +383,42 @@ revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" version = "v1.2.1" +[[projects]] + name = "github.com/tdewolff/minify" + packages = [ + ".", + "css", + "html", + "js", + "json", + "svg", + "xml" + ] + revision = "8d72a4127ae33b755e95bffede9b92e396267ce2" + version = "v2.3.5" + +[[projects]] + name = "github.com/tdewolff/parse" + packages = [ + ".", + "buffer", + "css", + "html", + "js", + "json", + "strconv", + "svg", + "xml" + ] + revision = "d739d6fccb0971177e06352fea02d3552625efb1" + version = "v2.3.3" + +[[projects]] + branch = "master" + name = "github.com/wellington/go-libsass" + packages = ["libs"] + revision = "615eaa47ef794d037c1906a0eb7bf85375a5decf" + [[projects]] name = "github.com/yosssi/ace" packages = ["."] @@ -431,6 +495,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "78b19539f7321429f217fc482de9e7cb4e2edd9b054ba8ec36b1e62bc4281b4f" + inputs-digest = "aaf909f54ae33c5a70f692e19e59834106bcbbe5d16724ff3998907734e32c0b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index c87b82823a7..8e6a614f2f1 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -16,6 +16,14 @@ branch = "master" name = "github.com/bep/gitmap" +[[constraint]] + branch = "master" + name = "github.com/bep/go-tocss" + +[[override]] + branch = "master" + name = "github.com/wellington/go-libsass" + [[constraint]] name = "github.com/chaseadamsio/goorgeous" version = "^1.1.0" @@ -149,3 +157,15 @@ [[constraint]] name = "github.com/bep/debounce" version = "^1.1.0" + +[[constraint]] + name = "github.com/tdewolff/minify" + version = "^2.3.5" + +[[constraint]] + branch = "master" + name = "github.com/BurntSushi/locker" + +[[constraint]] + branch = "master" + name = "github.com/mitchellh/hashstructure" diff --git a/commands/commandeer.go b/commands/commandeer.go index 4ca0c4be9b2..509d120d9b2 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -152,6 +152,8 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { doWithCommandeer, doWithConfig) + config.Set("isServer", running) + if err != nil { if mustHaveConfigFile { return err diff --git a/commands/hugo.go b/commands/hugo.go index 2b847ec95ea..85e0874cae3 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -552,8 +552,8 @@ func (c *commandeer) getDirList() ([]string, error) { // SymbolicWalk will log anny ERRORs // Also note that the Dirnames fetched below will contain any relevant theme // directories. - for _, contentDir := range c.hugo.PathSpec.BaseFs.AbsContentDirs { - _ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker) + for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker) } for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames { @@ -574,6 +574,10 @@ func (c *commandeer) getDirList() ([]string, error) { } } + for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker) + } + if len(nested) > 0 { for { diff --git a/common/errors/errors.go b/common/errors/errors.go new file mode 100644 index 00000000000..eff65ff92d9 --- /dev/null +++ b/common/errors/errors.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package errors contains common Hugo errors and error related utilities. +package errors + +import ( + "errors" +) + +// We will, at least to begin with, make some Hugo features (SCSS with libsass) optional, +// and this error is used to signal those situations. +var FeatureNotAvailableErr = errors.New("this feature is not available in your current Hugo version") diff --git a/create/content_test.go b/create/content_test.go index e9d46becfe9..f3bcc1dd561 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -88,6 +88,8 @@ func initViper(v *viper.Viper) { v.Set("i18nDir", "i18n") v.Set("theme", "sample") v.Set("archetypeDir", "archetypes") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") } func initFs(fs *hugofs.Fs) error { @@ -191,6 +193,7 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") fs := hugofs.NewMem(v) diff --git a/deps/deps.go b/deps/deps.go index d233025d303..0690d63f426 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -1,17 +1,18 @@ package deps import ( - "io/ioutil" - "log" - "os" "time" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/metrics" "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" jww "github.com/spf13/jwalterweatherman" @@ -42,6 +43,9 @@ type Deps struct { // The SourceSpec to use SourceSpec *source.SourceSpec `json:"-"` + // The Resource Spec to use + ResourceSpec *resource.Spec + // The configuration to use Cfg config.Provider `json:"-"` @@ -115,7 +119,7 @@ func New(cfg DepsCfg) (*Deps, error) { } if logger == nil { - logger = jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + logger = loggers.NewErrorLogger() } if fs == nil { @@ -129,6 +133,11 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } + resourceSpec, err := resource.NewSpec(ps, logger, cfg.MediaTypes) + if err != nil { + return nil, err + } + contentSpec, err := helpers.NewContentSpec(cfg.Language) if err != nil { return nil, err @@ -153,6 +162,7 @@ func New(cfg DepsCfg) (*Deps, error) { PathSpec: ps, ContentSpec: contentSpec, SourceSpec: sp, + ResourceSpec: resourceSpec, Cfg: cfg.Language, Language: cfg.Language, Timeout: time.Duration(timeoutms) * time.Millisecond, @@ -167,7 +177,8 @@ func New(cfg DepsCfg) (*Deps, error) { // ForLanguage creates a copy of the Deps with the language dependent // parts switched out. -func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) { +func (d Deps) ForLanguage(cfg DepsCfg) (*Deps, error) { + l := cfg.Language var err error d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs) @@ -180,6 +191,11 @@ func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) { return nil, err } + d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.Log, cfg.MediaTypes) + if err != nil { + return nil, err + } + d.Cfg = l d.Language = l @@ -212,6 +228,9 @@ type DepsCfg struct { // The configuration to use. Cfg config.Provider + // The media types configured. + MediaTypes media.Types + // Template handling. TemplateProvider ResourceProvider WithTemplate func(templ tpl.TemplateHandler) error diff --git a/helpers/general.go b/helpers/general.go index b442b1eb4f8..ab66376c32c 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -356,7 +356,7 @@ func MD5String(f string) string { // MD5FromFileFast creates a MD5 hash from the given file. It only reads parts of // the file for speed, so don't use it if the files are very subtly different. // It will not close the file. -func MD5FromFileFast(f afero.File) (string, error) { +func MD5FromFileFast(r io.ReadSeeker) (string, error) { const ( // Do not change once set in stone! maxChunks = 8 @@ -369,7 +369,7 @@ func MD5FromFileFast(f afero.File) (string, error) { for i := 0; i < maxChunks; i++ { if i > 0 { - _, err := f.Seek(seek, 0) + _, err := r.Seek(seek, 0) if err != nil { if err == io.EOF { break @@ -378,7 +378,7 @@ func MD5FromFileFast(f afero.File) (string, error) { } } - _, err := io.ReadAtLeast(f, buff, peekSize) + _, err := io.ReadAtLeast(r, buff, peekSize) if err != nil { if err == io.EOF || err == io.ErrUnexpectedEOF { h.Write(buff) diff --git a/helpers/path.go b/helpers/path.go index 76f13d653d7..bac4282d65d 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -222,12 +222,22 @@ func GetDottedRelativePath(inPath string) string { return dottedPath } +// ExtNoDelimiter takes a path and returns the extension, excluding the delmiter, i.e. "md". +func ExtNoDelimiter(in string) string { + return strings.TrimPrefix(Ext(in), ".") +} + // Ext takes a path and returns the extension, including the delmiter, i.e. ".md". func Ext(in string) string { _, ext := fileAndExt(in, fpb) return ext } +// PathAndExt is the same as FileAndExt, but it uses the path package. +func PathAndExt(in string) (string, string) { + return fileAndExt(in, pb) +} + // FileAndExt takes a path and returns the file and extension separated, // the extension including the delmiter, i.e. ".md". func FileAndExt(in string) (string, string) { diff --git a/helpers/path_test.go b/helpers/path_test.go index 2c6cb9f3768..c249a519dfe 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -78,6 +78,9 @@ func TestMakePathSanitized(t *testing.T) { v.Set("dataDir", "data") v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") v.Set("archetypeDir", "archetypes") l := langs.NewDefaultLanguage(v) @@ -475,6 +478,7 @@ func createTempDirWithNonZeroLengthFiles() (string, error) { return "", fileErr } byteString := []byte("byteString") + fileErr = ioutil.WriteFile(f.Name(), byteString, 0644) if fileErr != nil { // delete the file @@ -585,6 +589,11 @@ func TestAbsPathify(t *testing.T) { } +func TestExtNoDelimiter(t *testing.T) { + assert := require.New(t) + assert.Equal("json", ExtNoDelimiter(filepath.FromSlash("/my/data.json"))) +} + func TestFilename(t *testing.T) { type test struct { input, expected string diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index fda1c9ea205..c9da4f12919 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -38,6 +38,9 @@ func newTestCfg() *viper.Viper { v.Set("dataDir", "data") v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") v.Set("archetypeDir", "archetypes") return v } diff --git a/hugofs/basepath_real_filename_fs.go b/hugofs/basepath_real_filename_fs.go new file mode 100644 index 00000000000..d0c56df74e9 --- /dev/null +++ b/hugofs/basepath_real_filename_fs.go @@ -0,0 +1,84 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + + "github.com/spf13/afero" +) + +// RealFilenameInfo is a thin wrapper around os.FileInfo adding the real filename. +type RealFilenameInfo interface { + os.FileInfo + + // This is the real filename to the file in the underlying filesystem. + RealFilename() string +} + +type realFilenameInfo struct { + os.FileInfo + realFilename string +} + +func (f *realFilenameInfo) RealFilename() string { + return f.realFilename +} + +func NewBasePathRealFilenameFs(base *afero.BasePathFs) *BasePathRealFilenameFs { + return &BasePathRealFilenameFs{BasePathFs: base} +} + +// This is a thin wrapper around afero.BasePathFs that provides the real filename +// in Stat and LstatIfPossible. +type BasePathRealFilenameFs struct { + *afero.BasePathFs +} + +func (b *BasePathRealFilenameFs) Stat(name string) (os.FileInfo, error) { + fi, err := b.BasePathFs.Stat(name) + if err != nil { + return nil, err + } + + if _, ok := fi.(RealFilenameInfo); ok { + return fi, nil + } + + filename, err := b.RealPath(name) + if err != nil { + return nil, &os.PathError{Op: "stat", Path: name, Err: err} + } + + return &realFilenameInfo{FileInfo: fi, realFilename: filename}, nil +} + +func (b *BasePathRealFilenameFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + + fi, ok, err := b.BasePathFs.LstatIfPossible(name) + if err != nil { + return nil, false, err + } + + if _, ok := fi.(RealFilenameInfo); ok { + return fi, ok, nil + } + + filename, err := b.RealPath(name) + if err != nil { + return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} + } + + return &realFilenameInfo{FileInfo: fi, realFilename: filename}, ok, nil +} diff --git a/hugolib/alias_test.go b/hugolib/alias_test.go index 04c5b4358b1..da1b80b7007 100644 --- a/hugolib/alias_test.go +++ b/hugolib/alias_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/hugolib/case_insensitive_test.go b/hugolib/case_insensitive_test.go index f3ba5f933ac..df147e4ad4f 100644 --- a/hugolib/case_insensitive_test.go +++ b/hugolib/case_insensitive_test.go @@ -134,7 +134,7 @@ Partial Site: {{ .Site.Params.COLOR }}|{{ .Site.Params.COLORS.YELLOW }} func TestCaseInsensitiveConfigurationVariations(t *testing.T) { t.Parallel() - // See issues 2615, 1129, 2590 and maybe some others + // See issuess 2615, 1129, 2590 and maybe some others // Also see 2598 // // Viper is now, at least for the Hugo part, case insensitive diff --git a/hugolib/config.go b/hugolib/config.go index dec5b870df8..87f97f3a51d 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -411,6 +411,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("metaDataFormat", "toml") v.SetDefault("contentDir", "content") v.SetDefault("layoutDir", "layouts") + v.SetDefault("assetDir", "assets") v.SetDefault("staticDir", "static") v.SetDefault("resourceDir", "resources") v.SetDefault("archetypeDir", "archetypes") diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index deecd69a5da..2aa713ae68b 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -28,7 +28,6 @@ import ( "fmt" - "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/hugolib/paths" "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" @@ -45,20 +44,10 @@ var filePathSeparator = string(filepath.Separator) // to underline that even if they can be composites, they all have a base path set to a specific // resource folder, e.g "/my-project/content". So, no absolute filenames needed. type BaseFs struct { - // TODO(bep) make this go away - AbsContentDirs []types.KeyValueStr - - // The filesystem used to capture content. This can be a composite and - // language aware file system. - ContentFs afero.Fs // SourceFilesystems contains the different source file systems. *SourceFilesystems - // The filesystem used to store resources (processed images etc.). - // This usually maps to /my-project/resources. - ResourcesFs afero.Fs - // The filesystem used to publish the rendered site. // This usually maps to /my-project/public. PublishFs afero.Fs @@ -71,35 +60,31 @@ type BaseFs struct { // RelContentDir tries to create a path relative to the content root from // the given filename. The return value is the path and language code. -func (b *BaseFs) RelContentDir(filename string) (string, string) { - for _, dir := range b.AbsContentDirs { - if strings.HasPrefix(filename, dir.Value) { - rel := strings.TrimPrefix(filename, dir.Value) - return strings.TrimPrefix(rel, filePathSeparator), dir.Key +func (b *BaseFs) RelContentDir(filename string) string { + for _, dirname := range b.SourceFilesystems.Content.Dirnames { + if strings.HasPrefix(filename, dirname) { + rel := strings.TrimPrefix(filename, dirname) + return strings.TrimPrefix(rel, filePathSeparator) } } // Either not a content dir or already relative. - return filename, "" -} - -// IsContent returns whether the given filename is in the content filesystem. -func (b *BaseFs) IsContent(filename string) bool { - for _, dir := range b.AbsContentDirs { - if strings.HasPrefix(filename, dir.Value) { - return true - } - } - return false + return filename } // SourceFilesystems contains the different source file systems. These can be // composite file systems (theme and project etc.), and they have all root // set to the source type the provides: data, i18n, static, layouts. type SourceFilesystems struct { + Content *SourceFilesystem Data *SourceFilesystem I18n *SourceFilesystem Layouts *SourceFilesystem Archetypes *SourceFilesystem + Assets *SourceFilesystem + Resources *SourceFilesystem + + // This is a unified read-only view of the project's and themes' workdir. + Work *SourceFilesystem // When in multihost we have one static filesystem per language. The sync // static files is currently done outside of the Hugo build (where there is @@ -112,8 +97,14 @@ type SourceFilesystems struct { // i18n, layouts, static) and additional metadata to be able to use that filesystem // in server mode. type SourceFilesystem struct { + // This is a virtual composite filesystem. It expects path relative to a context. Fs afero.Fs + // This is the base source filesystem. In real Hugo, this will be the OS filesystem. + // Use this if you need to resolve items in Dirnames below. + SourceFs afero.Fs + + // Dirnames is absolute filenames to the directories in this filesystem. Dirnames []string // When syncing a source folder to the target (e.g. /public), this may @@ -122,6 +113,50 @@ type SourceFilesystem struct { PublishFolder string } +// ContentStaticAssetFs will create a new composite filesystem from the content, +// static, and asset filesystems. The site language is needed to pick the correct static filesystem. +// The order is content, static and then assets. +// TODO(bep) check usage +func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs { + staticFs := s.StaticFs(lang) + + base := afero.NewCopyOnWriteFs(s.Assets.Fs, staticFs) + return afero.NewCopyOnWriteFs(base, s.Content.Fs) + +} + +// StaticFs returns the static filesystem for the given language. +// This can be a composite filesystem. +func (s SourceFilesystems) StaticFs(lang string) afero.Fs { + var staticFs afero.Fs = hugofs.NoOpFs + + if fs, ok := s.Static[lang]; ok { + staticFs = fs.Fs + } else if fs, ok := s.Static[""]; ok { + staticFs = fs.Fs + } + + return staticFs +} + +// StatResource looks for a resource in these filesystems in order: static, assets and finally content. +// If found in any of them, it returns FileInfo and the relevant filesystem. +// Any non os.IsNotExist error will be returned. +// An os.IsNotExist error wil be returned only if all filesystems return such an error. +// Note that if we only wanted to find the file, we could create a composite Afero fs, +// but we also need to know which filesystem root it lives in. +func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) { + for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} { + fs = fsToCheck + fi, err = fs.Stat(filename) + if err == nil || !os.IsNotExist(err) { + return + } + } + // Not found. + return +} + // IsStatic returns true if the given filename is a member of one of the static // filesystems. func (s SourceFilesystems) IsStatic(filename string) bool { @@ -133,6 +168,11 @@ func (s SourceFilesystems) IsStatic(filename string) bool { return false } +// IsContent returns true if the given filename is a member of the content filesystem. +func (s SourceFilesystems) IsContent(filename string) bool { + return s.Content.Contains(filename) +} + // IsLayout returns true if the given filename is a member of the layouts filesystem. func (s SourceFilesystems) IsLayout(filename string) bool { return s.Layouts.Contains(filename) @@ -171,6 +211,18 @@ func (d *SourceFilesystem) MakePathRelative(filename string) string { return "" } +func (d *SourceFilesystem) RealFilename(rel string) string { + fi, err := d.Fs.Stat(rel) + if err != nil { + return rel + } + if realfi, ok := fi.(hugofs.RealFilenameInfo); ok { + return realfi.RealFilename() + } + + return rel +} + // Contains returns whether the given filename is a member of the current filesystem. func (d *SourceFilesystem) Contains(filename string) bool { for _, dir := range d.Dirnames { @@ -181,6 +233,20 @@ func (d *SourceFilesystem) Contains(filename string) bool { return false } +// RealDirs gets a list of absolute paths to directorys starting from the given +// path. +func (d *SourceFilesystem) RealDirs(from string) []string { + var dirnames []string + for _, dir := range d.Dirnames { + dirname := filepath.Join(dir, from) + + if _, err := hugofs.Os.Stat(dirname); err == nil { + dirnames = append(dirnames, dirname) + } + } + return dirnames +} + // WithBaseFs allows reuse of some potentially expensive to create parts that remain // the same across sites/languages. func WithBaseFs(b *BaseFs) func(*BaseFs) error { @@ -191,11 +257,15 @@ func WithBaseFs(b *BaseFs) func(*BaseFs) error { } } +func newRealBase(base afero.Fs) afero.Fs { + return hugofs.NewBasePathRealFilenameFs(base.(*afero.BasePathFs)) + +} + // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { fs := p.Fs - resourcesFs := afero.NewBasePathFs(fs.Source, p.AbsResourcesDir) publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages) @@ -209,17 +279,14 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { if i == j { continue } - if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { + if strings.HasPrefix(d1, d2) || strings.HasPrefix(d2, d1) { return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) } } } b := &BaseFs{ - AbsContentDirs: absContentDirs, - ContentFs: contentFs, - ResourcesFs: resourcesFs, - PublishFs: publishFs, + PublishFs: publishFs, } for _, opt := range options { @@ -234,6 +301,12 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { return nil, err } + sourceFilesystems.Content = &SourceFilesystem{ + SourceFs: fs.Source, + Fs: contentFs, + Dirnames: absContentDirs, + } + b.SourceFilesystems = sourceFilesystems b.themeFs = builder.themeFs b.AbsThemeDirs = builder.absThemeDirs @@ -281,18 +354,49 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { } b.result.I18n = sfs - sfs, err = b.createFs("layoutDir", "layouts") + sfs, err = b.createFs(false, true, "layoutDir", "layouts") if err != nil { return nil, err } b.result.Layouts = sfs - sfs, err = b.createFs("archetypeDir", "archetypes") + sfs, err = b.createFs(false, true, "archetypeDir", "archetypes") if err != nil { return nil, err } b.result.Archetypes = sfs + sfs, err = b.createFs(false, true, "assetDir", "assets") + if err != nil { + return nil, err + } + b.result.Assets = sfs + + sfs, err = b.createFs(true, false, "resourceDir", "resources") + if err != nil { + return nil, err + } + + if b.p.AbsTempResourcesDir != "" { + // TODO(bep) resource + fmt.Println(">>> WRITE RESOURCES TO", b.p.AbsTempResourcesDir) + // Write to temp only. This is to avoid filling up the project on + // SCSS changes etc. + layerResourcesFs := newRealBase(afero.NewBasePathFs(b.p.Fs.Source, b.p.AbsTempResourcesDir)) + sfs.Fs = afero.NewCopyOnWriteFs(sfs.Fs, layerResourcesFs) + sfs.Dirnames = append(sfs.Dirnames, b.p.AbsTempResourcesDir) + } + + b.result.Resources = sfs + + err = b.createStaticFs() + + sfs, err = b.createFs(false, true, "", "") + if err != nil { + return nil, err + } + b.result.Work = sfs + err = b.createStaticFs() if err != nil { return nil, err @@ -301,23 +405,38 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { return b.result, nil } -func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{} - dir := b.p.Cfg.GetString(dirKey) - if dir == "" { - return s, fmt.Errorf("config %q not set", dirKey) +func (b *sourceFilesystemsBuilder) createFs( + mkdir bool, + readOnly bool, + dirKey, themeFolder string) (*SourceFilesystem, error) { + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + } + var dir string + if dirKey != "" { + dir = b.p.Cfg.GetString(dirKey) + if dir == "" { + return s, fmt.Errorf("config %q not set", dirKey) + } } var fs afero.Fs absDir := b.p.AbsPathify(dir) - if b.existsInSource(absDir) { - fs = afero.NewBasePathFs(b.p.Fs.Source, absDir) + existsInSource := b.existsInSource(absDir) + if !existsInSource && mkdir { + // We really need this directory. Make it. + if err := b.p.Fs.Source.MkdirAll(absDir, 0777); err == nil { + existsInSource = true + } + } + if existsInSource { + fs = newRealBase(afero.NewBasePathFs(b.p.Fs.Source, absDir)) s.Dirnames = []string{absDir} } if b.hasTheme { - themeFolderFs := afero.NewBasePathFs(b.themeFs, themeFolder) + themeFolderFs := newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)) if fs == nil { fs = themeFolderFs } else { @@ -334,8 +453,10 @@ func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*Source if fs == nil { s.Fs = hugofs.NoOpFs - } else { + } else if readOnly { s.Fs = afero.NewReadOnlyFs(fs) + } else { + s.Fs = fs } return s, nil @@ -344,7 +465,9 @@ func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*Source // Used for data, i18n -- we cannot use overlay filsesystems for those, but we need // to keep a strict order. func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{} + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + } projectDir := b.p.Cfg.GetString(dirKey) if projectDir == "" { @@ -396,7 +519,9 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error { if isMultihost { for _, l := range b.p.Languages { - s := &SourceFilesystem{PublishFolder: l.Lang} + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + PublishFolder: l.Lang} staticDirs := removeDuplicatesKeepRight(getStaticDirs(l)) if len(staticDirs) == 0 { continue @@ -424,7 +549,10 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error { return nil } - s := &SourceFilesystem{} + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + } + var staticDirs []string for _, l := range b.p.Languages { @@ -451,7 +579,7 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error { if b.hasTheme { themeFolder := "static" - fs = afero.NewCopyOnWriteFs(afero.NewBasePathFs(b.themeFs, themeFolder), fs) + fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs) for _, absThemeDir := range b.absThemeDirs { s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) } @@ -484,7 +612,7 @@ func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { func createContentFs(fs afero.Fs, workingDir, defaultContentLanguage string, - languages langs.Languages) (afero.Fs, []types.KeyValueStr, error) { + languages langs.Languages) (afero.Fs, []string, error) { var contentLanguages langs.Languages var contentDirSeen = make(map[string]bool) @@ -511,7 +639,7 @@ func createContentFs(fs afero.Fs, } - var absContentDirs []types.KeyValueStr + var absContentDirs []string fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) return fs, absContentDirs, err @@ -522,7 +650,7 @@ func createContentOverlayFs(source afero.Fs, workingDir string, languages langs.Languages, languageSet map[string]bool, - absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { + absContentDirs *[]string) (afero.Fs, error) { if len(languages) == 0 { return source, nil } @@ -548,7 +676,7 @@ func createContentOverlayFs(source afero.Fs, return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir) } - *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) + *absContentDirs = append(*absContentDirs, absContentDir) overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) if len(languages) == 1 { @@ -597,10 +725,10 @@ func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) { } if len(absPaths) == 1 { - return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])), nil + return afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))), nil } - base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) + base := afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))) overlay, err := createOverlayFs(source, absPaths[1:]) if err != nil { return nil, err diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index ea09cd8fd8b..1af3826c860 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -60,6 +60,10 @@ theme = ["atheme"] setConfigAndWriteSomeFilesTo(fs.Source, v, "staticDir", "mystatic", 6) setConfigAndWriteSomeFilesTo(fs.Source, v, "dataDir", "mydata", 7) setConfigAndWriteSomeFilesTo(fs.Source, v, "archetypeDir", "myarchetypes", 8) + setConfigAndWriteSomeFilesTo(fs.Source, v, "assetDir", "myassets", 9) + setConfigAndWriteSomeFilesTo(fs.Source, v, "resourceDir", "myrsesource", 10) + + v.Set("publishDir", "public") p, err := paths.New(fs, v) assert.NoError(err) @@ -88,12 +92,15 @@ theme = ["atheme"] _, err = ff.Readdirnames(-1) assert.NoError(err) - checkFileCount(bfs.ContentFs, "", assert, 3) + checkFileCount(bfs.Content.Fs, "", assert, 3) checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes checkFileCount(bfs.Layouts.Fs, "", assert, 5) checkFileCount(bfs.Static[""].Fs, "", assert, 6) checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes checkFileCount(bfs.Archetypes.Fs, "", assert, 8) + checkFileCount(bfs.Assets.Fs, "", assert, 9) + checkFileCount(bfs.Resources.Fs, "", assert, 10) + checkFileCount(bfs.Work.Fs, "", assert, 57) assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames) @@ -103,13 +110,12 @@ theme = ["atheme"] assert.True(bfs.IsStatic(filepath.Join(workingDir, "mystatic", "file1.txt"))) contentFilename := filepath.Join(workingDir, "mycontent", "file1.txt") assert.True(bfs.IsContent(contentFilename)) - rel, _ := bfs.RelContentDir(contentFilename) + rel := bfs.RelContentDir(contentFilename) assert.Equal("file1.txt", rel) } -func TestNewBaseFsEmpty(t *testing.T) { - assert := require.New(t) +func createConfig() *viper.Viper { v := viper.New() v.Set("contentDir", "mycontent") v.Set("i18nDir", "myi18n") @@ -117,18 +123,90 @@ func TestNewBaseFsEmpty(t *testing.T) { v.Set("dataDir", "mydata") v.Set("layoutDir", "mylayouts") v.Set("archetypeDir", "myarchetypes") + v.Set("assetDir", "myassets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + return v +} + +func TestNewBaseFsEmpty(t *testing.T) { + assert := require.New(t) + v := createConfig() fs := hugofs.NewMem(v) p, err := paths.New(fs, v) + assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) assert.NotNil(bfs) assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs) assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs) assert.Equal(hugofs.NoOpFs, bfs.Data.Fs) + assert.Equal(hugofs.NoOpFs, bfs.Assets.Fs) assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs) - assert.NotNil(hugofs.NoOpFs, bfs.ContentFs) - assert.NotNil(hugofs.NoOpFs, bfs.Static) + assert.NotNil(bfs.Work.Fs) + assert.NotNil(bfs.Content.Fs) + assert.NotNil(bfs.Static) +} + +func TestRealDirs(t *testing.T) { + assert := require.New(t) + v := createConfig() + fs := hugofs.NewDefault(v) + sfs := fs.Source + + root, err := afero.TempDir(sfs, "", "realdir") + assert.NoError(err) + themesDir, err := afero.TempDir(sfs, "", "themesDir") + assert.NoError(err) + defer func() { + os.RemoveAll(root) + os.RemoveAll(themesDir) + }() + + v.Set("workingDir", root) + v.Set("contentDir", "content") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + v.Set("themesDir", themesDir) + v.Set("theme", "mytheme") + + assert.NoError(sfs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf1"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf2"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(root, "resources"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "resources"), 0755)) + + assert.NoError(sfs.MkdirAll(filepath.Join(root, "myassets", "js", "f2"), 0755)) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf1", "a1.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf2", "a3.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "a2.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2", "a3.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3", "a4.scss")), []byte("content"), 0755) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "resources", "t1.txt")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "resources", "p1.txt")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "resources", "p2.txt")), []byte("content"), 0755) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0755) + + p, err := paths.New(fs, v) + assert.NoError(err) + bfs, err := NewBase(p) + assert.NoError(err) + assert.NotNil(bfs) + checkFileCount(bfs.Assets.Fs, "", assert, 6) + + realDirs := bfs.Assets.RealDirs("scss") + assert.Equal(2, len(realDirs)) + assert.Equal(filepath.Join(root, "myassets/scss"), realDirs[0]) + assert.Equal(filepath.Join(themesDir, "mytheme/assets/scss"), realDirs[len(realDirs)-1]) + + checkFileCount(bfs.Resources.Fs, "", assert, 3) + } func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index a0ac72d67ce..8cb3cf2fd8c 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -21,8 +21,6 @@ import ( "strings" "sync" - "github.com/gohugoio/hugo/resource" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/langs" @@ -182,8 +180,10 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { continue } + cfg.Language = s.Language + cfg.MediaTypes = s.mediaTypesConfig + if d == nil { - cfg.Language = s.Language cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate) var err error @@ -200,7 +200,7 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { } } else { - d, err = d.ForLanguage(s.Language) + d, err = d.ForLanguage(cfg) if err != nil { return err } @@ -208,11 +208,6 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { s.Deps = d } - s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig) - if err != nil { - return err - } - } return nil @@ -701,7 +696,7 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu defer m.mu.RUnlock() // Bundles share resources, so we need to start from the virtual root. - relPath, _ := m.pathSpec.RelContentDir(filename) + relPath := m.pathSpec.RelContentDir(filename) dir, name := filepath.Split(relPath) if !strings.HasSuffix(dir, helpers.FilePathSeparator) { dir += helpers.FilePathSeparator diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index cf7c514f699..b51f1eee055 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -461,7 +461,7 @@ func TestMultiSitesRebuild(t *testing.T) { b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Shortcode: Bonjour") b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello") - contentFs := b.H.BaseFs.ContentFs + contentFs := b.H.BaseFs.Content.Fs for i, this := range []struct { preFunc func(t *testing.T) diff --git a/hugolib/page.go b/hugolib/page.go index 322660647e8..fb570d6e1fd 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -21,6 +21,8 @@ import ( "reflect" "unicode" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/langs" @@ -239,7 +241,8 @@ type Page struct { permalink string relPermalink string - // relative target path without extension and any base path element from the baseURL. + // relative target path without extension and any base path element + // from the baseURL or the language code. // This is used to construct paths in the page resources. relTargetPathBase string // Is set to a forward slashed path if this is a Page resources living in a folder below its owner. @@ -272,8 +275,8 @@ type Page struct { targetPathDescriptorPrototype *targetPathDescriptor } -func stackTrace() string { - trace := make([]byte, 2000) +func stackTrace(lenght int) string { + trace := make([]byte, lenght) runtime.Stack(trace, true) return string(trace) } @@ -476,6 +479,10 @@ func (p *Page) BundleType() string { return "" } +func (p *Page) MediaType() media.Type { + return media.OctetType +} + type Source struct { Frontmatter []byte Content []byte diff --git a/hugolib/page_bundler.go b/hugolib/page_bundler.go index e55e0a92be7..9ebfe1b8870 100644 --- a/hugolib/page_bundler.go +++ b/hugolib/page_bundler.go @@ -144,7 +144,7 @@ func (s *siteContentProcessor) process(ctx context.Context) error { return nil } for _, file := range files { - f, err := s.site.BaseFs.ContentFs.Open(file.Filename()) + f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) if err != nil { return fmt.Errorf("failed to open assets file: %s", err) } diff --git a/hugolib/page_bundler_capture_test.go b/hugolib/page_bundler_capture_test.go index 14d8a436843..96d113bf746 100644 --- a/hugolib/page_bundler_capture_test.go +++ b/hugolib/page_bundler_capture_test.go @@ -91,7 +91,7 @@ func TestPageBundlerCaptureSymlinks(t *testing.T) { assert := require.New(t) ps, workDir := newTestBundleSymbolicSources(t) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) fileStore := &storeFilenames{} logger := loggers.NewErrorLogger() @@ -137,7 +137,7 @@ func TestPageBundlerCaptureBasic(t *testing.T) { ps, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) fileStore := &storeFilenames{} @@ -183,7 +183,7 @@ func TestPageBundlerCaptureMultilingual(t *testing.T) { ps, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) fileStore := &storeFilenames{} c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) diff --git a/hugolib/page_bundler_handlers.go b/hugolib/page_bundler_handlers.go index eca324294f8..e0eac3ac49d 100644 --- a/hugolib/page_bundler_handlers.go +++ b/hugolib/page_bundler_handlers.go @@ -326,9 +326,14 @@ func (c *contentHandlers) createResource() contentHandler { return notHandled } - resource, err := c.s.resourceSpec.NewResourceFromFilename( - ctx.parentPage.subResourceTargetPathFactory, - ctx.source.Filename(), ctx.target) + resource, err := c.s.ResourceSpec.New( + resource.ResourceSourceDescriptor{ + TargetPathBuilder: ctx.parentPage.subResourceTargetPathFactory, + SourceFile: ctx.source, + RelTargetFilename: ctx.target, + URLBase: c.s.GetURLLanguageBasePath(), + TargetPathBase: c.s.GetTargetLanguageBasePath(), + }) return handlerResult{err: err, handled: true, resource: resource} } @@ -336,7 +341,7 @@ func (c *contentHandlers) createResource() contentHandler { func (c *contentHandlers) copyFile() contentHandler { return func(ctx *handlerContext) handlerResult { - f, err := c.s.BaseFs.ContentFs.Open(ctx.source.Filename()) + f, err := c.s.BaseFs.Content.Fs.Open(ctx.source.Filename()) if err != nil { err := fmt.Errorf("failed to open file in copyFile: %s", err) return handlerResult{err: err} diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go index 3af553ec3f7..811dbf56fe8 100644 --- a/hugolib/page_bundler_test.go +++ b/hugolib/page_bundler_test.go @@ -37,7 +37,6 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/resource" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -158,7 +157,6 @@ func TestPageBundlerSiteRegular(t *testing.T) { altFormat := leafBundle1.OutputFormats().Get("CUSTOMO") assert.NotNil(altFormat) - assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename()) assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink()) th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") diff --git a/hugolib/page_collections.go b/hugolib/page_collections.go index 74f7d608ced..8395502f576 100644 --- a/hugolib/page_collections.go +++ b/hugolib/page_collections.go @@ -220,6 +220,6 @@ func (c *PageCollections) clearResourceCacheForPage(page *Page) { dir := path.Dir(first.RelPermalink()) dir = strings.TrimPrefix(dir, page.LanguagePrefix()) // This is done to keep the memory usage in check when doing live reloads. - page.s.resourceSpec.DeleteCacheByPrefix(dir) + page.s.ResourceSpec.DeleteCacheByPrefix(dir) } } diff --git a/hugolib/page_output.go b/hugolib/page_output.go index c1550ccd14a..8e10f8f4fe4 100644 --- a/hugolib/page_output.go +++ b/hugolib/page_output.go @@ -265,7 +265,7 @@ func (p *PageOutput) renderResources() error { // mode when the same resource is member of different page bundles. p.deleteResource(i) } else { - p.s.Log.ERROR.Printf("Failed to publish %q for page %q: %s", src.AbsSourceFilename(), p.pathOrTitle(), err) + p.s.Log.ERROR.Printf("Failed to publish Resource for page %q: %s", p.pathOrTitle(), err) } } else { p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Files) diff --git a/hugolib/page_paths.go b/hugolib/page_paths.go index 4d64f4c1488..1b2d00ad5c3 100644 --- a/hugolib/page_paths.go +++ b/hugolib/page_paths.go @@ -139,7 +139,11 @@ func (p *Page) initURLs() error { return err } - p.relTargetPathBase = strings.TrimSuffix(target, f.MediaType.FullSuffix()) + p.relTargetPathBase = strings.TrimPrefix(strings.TrimSuffix(target, f.MediaType.FullSuffix()), "/") + if prefix := p.s.GetLanguagePrefix(); prefix != "" { + // Any language code in the path will be added later. + p.relTargetPathBase = strings.TrimPrefix(p.relTargetPathBase, prefix+"/") + } p.relPermalink = p.s.PathSpec.PrependBasePath(rel) p.layoutDescriptor = p.createLayoutDescriptor() return nil diff --git a/hugolib/page_paths_test.go b/hugolib/page_paths_test.go index 149505ee44f..3ca500f179a 100644 --- a/hugolib/page_paths_test.go +++ b/hugolib/page_paths_test.go @@ -27,7 +27,7 @@ import ( func TestPageTargetPath(t *testing.T) { - pathSpec := newTestDefaultPathSpec() + pathSpec := newTestDefaultPathSpec(t) noExtNoDelimMediaType := media.TextType noExtNoDelimMediaType.Suffix = "" diff --git a/hugolib/paths/baseURL.go b/hugolib/paths/baseURL.go index 9cb5627ba41..de36c863640 100644 --- a/hugolib/paths/baseURL.go +++ b/hugolib/paths/baseURL.go @@ -27,13 +27,21 @@ type BaseURL struct { } func (b BaseURL) String() string { - return b.urlStr + if b.urlStr != "" { + return b.urlStr + } + return b.url.String() } func (b BaseURL) Path() string { return b.url.Path } +// HostURL returns the URL to the host root without any path elements. +func (b BaseURL) HostURL() string { + return strings.TrimSuffix(b.String(), b.Path()) +} + // WithProtocol returns the BaseURL prefixed with the given protocol. // The Protocol is normally of the form "scheme://", i.e. "webcal://". func (b BaseURL) WithProtocol(protocol string) (string, error) { diff --git a/hugolib/paths/baseURL_test.go b/hugolib/paths/baseURL_test.go index af1d2e38d80..382a18314b2 100644 --- a/hugolib/paths/baseURL_test.go +++ b/hugolib/paths/baseURL_test.go @@ -58,4 +58,9 @@ func TestBaseURL(t *testing.T) { require.NoError(t, err) require.Equal(t, "", b.String()) + // BaseURL with sub path + b, err = newBaseURLFromString("http://example.com/sub") + require.NoError(t, err) + require.Equal(t, "http://example.com/sub", b.String()) + require.Equal(t, "http://example.com", b.HostURL()) } diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index cf8792e5a9a..18ee771a87a 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -18,6 +18,8 @@ import ( "path/filepath" "strings" + "github.com/spf13/afero" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/langs" @@ -39,11 +41,16 @@ type Paths struct { // Directories // TODO(bep) when we have trimmed down mos of the dirs usage outside of this package, make // these into an interface. - ContentDir string - ThemesDir string - WorkingDir string - AbsResourcesDir string - AbsPublishDir string + ContentDir string + ThemesDir string + WorkingDir string + + // Directories to store Resource related artifacts. AbsTempResourcesDir + // is used to write generated content to when running in server mode. + AbsTempResourcesDir string + AbsResourcesDir string + + AbsPublishDir string // pagination path handling PaginatePath string @@ -79,12 +86,33 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) } - // TODO(bep) + isServer := cfg.GetBool("isServer") + contentDir := cfg.GetString("contentDir") workingDir := cfg.GetString("workingDir") resourceDir := cfg.GetString("resourceDir") publishDir := cfg.GetString("publishDir") + if contentDir == "" { + return nil, fmt.Errorf("contentDir not set") + } + if resourceDir == "" { + return nil, fmt.Errorf("resourceDir not set") + } + if publishDir == "" { + return nil, fmt.Errorf("publishDir not set") + } + + var tempResourcesDir string + if isServer { + // Write resources to a temp dir + var err error + tempResourcesDir, err = afero.TempDir(fs.Source, "", "hugo_resources") + if err != nil { + return nil, fmt.Errorf("failed to create temporary directory to store resources: %s", err) + } + } + defaultContentLanguage := cfg.GetString("defaultContentLanguage") var ( @@ -137,8 +165,9 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { ThemesDir: cfg.GetString("themesDir"), WorkingDir: workingDir, - AbsResourcesDir: absResourcesDir, - AbsPublishDir: absPublishDir, + AbsResourcesDir: absResourcesDir, + AbsTempResourcesDir: tempResourcesDir, + AbsPublishDir: absPublishDir, themes: config.GetStringSlicePreserveString(cfg, "theme"), @@ -183,6 +212,21 @@ func (p *Paths) Themes() []string { return p.themes } +func (p *Paths) GetTargetLanguageBasePath() string { + if p.Languages.IsMultihost() { + // In a multihost configuration all assets will be published below the language code. + return p.Lang() + } + return p.GetLanguagePrefix() +} + +func (p *Paths) GetURLLanguageBasePath() string { + if p.Languages.IsMultihost() { + return "" + } + return p.GetLanguagePrefix() +} + func (p *Paths) GetLanguagePrefix() string { if !p.multilingual { return "" diff --git a/hugolib/paths/paths_test.go b/hugolib/paths/paths_test.go index 6cadc747f6a..3bd445b8bc6 100644 --- a/hugolib/paths/paths_test.go +++ b/hugolib/paths/paths_test.go @@ -30,6 +30,10 @@ func TestNewPaths(t *testing.T) { v.Set("defaultContentLanguageInSubdir", true) v.Set("defaultContentLanguage", "no") v.Set("multilingual", true) + v.Set("contentDir", "content") + v.Set("workingDir", "work") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") p, err := New(fs, v) assert.NoError(err) diff --git a/hugolib/prune_resources.go b/hugolib/prune_resources.go index e9d2bf96e05..8076126a6d5 100644 --- a/hugolib/prune_resources.go +++ b/hugolib/prune_resources.go @@ -25,9 +25,9 @@ import ( // GC requires a build first. func (h *HugoSites) GC() (int, error) { s := h.Sites[0] - fs := h.PathSpec.BaseFs.ResourcesFs + fs := h.PathSpec.BaseFs.Resources.Fs - imageCacheDir := s.resourceSpec.GenImagePath + imageCacheDir := s.ResourceSpec.GenImagePath if len(imageCacheDir) < 10 { panic("invalid image cache") } @@ -35,7 +35,7 @@ func (h *HugoSites) GC() (int, error) { isInUse := func(filename string) bool { key := strings.TrimPrefix(filename, imageCacheDir) for _, site := range h.Sites { - if site.resourceSpec.IsInCache(key) { + if site.ResourceSpec.IsInCache(key) { return true } } diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go new file mode 100644 index 00000000000..a0d6207593b --- /dev/null +++ b/hugolib/resource_chain_test.go @@ -0,0 +1,162 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/resource/tocss/scss" +) + +func TestResourceChain(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + shouldRun func() bool + prepare func(b *sitesBuilder) + verify func(b *sitesBuilder) + }{ + {"tocss", func() bool { return scss.Supports() }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $scss := resources.Open "scss/styles2.scss" | toCSS }} +{{ $scssMin := resources.Open "scss/styles2.scss" | toCSS | minify }} +T1: Len Content: {{ len $scss.Content }}|RelPermalink: {{ $scss.RelPermalink }}|Permalink: {{ $scss.Permalink }}|MediaType: {{ $scss.MediaType.Type }} +T2: Content: {{ $scssMin.Content }}|RelPermalink: {{ $scssMin.RelPermalink }} +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `T1: Len Content: 24|RelPermalink: /scss/styles2.css|Permalink: http://example.com/scss/styles2.css|MediaType: text/css`) + b.AssertFileContent("public/index.html", `T2: Content: body{color:#333}|RelPermalink: /scss/styles2.min.css`) + + }}, + + {"minify", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +Min CSS: {{ ( resources.Open "css/styles1.css" | minify ).Content }} +Min JS: {{ ( resources.Open "js/script1.js" | resources.Minify ).Content | safeJS }} +Min JSON: {{ ( resources.Open "mydata/json1.json" | resources.Minify ).Content | safeHTML }} +Min XML: {{ ( resources.Open "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }} +Min SVG: {{ ( resources.Open "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }} +Min SVG again: {{ ( resources.Open "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }} +Min HTML: {{ ( resources.Open "mydata/html1.html" | resources.Minify ).Content | safeHTML }} + + +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`) + b.AssertFileContent("public/index.html", `Min JS: var x;x=5;document.getElementById("demo").innerHTML=x*10;`) + b.AssertFileContent("public/index.html", `Min JSON: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`) + b.AssertFileContent("public/index.html", `Min XML: Hugo Rocks!`) + b.AssertFileContent("public/index.html", `Min SVG: `) + b.AssertFileContent("public/index.html", `Min SVG again: `) + b.AssertFileContent("public/index.html", `Min HTML: Cool`) + }}, + + {"concat", func() bool { return scss.Supports() }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $scss := resources.Open "scss/styles2.scss" | toCSS }} +{{ $scss := slice $scss $scss | resources.ConcatTo "myfolder/concat.css" }} +T2: Len Content: {{ len $scss.Content }}|RelPermalink: {{ $scss.RelPermalink }}|Permalink: {{ $scss.Permalink }}|MediaType: {{ $scss.MediaType.Type }} +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `T2: Len Content: 84|RelPermalink: /myfolder/concat.css|Permalink: http://example.com/myfolder/concat.css|MediaType: text/css`) + b.AssertFileContent("public/myfolder/concat.css", "\n$color: #333;\n\nbody {\n color: $color;\n}\n\n$color: #333;\n\nbody {\n color: $color;\n}\n") + }}, + + {"fromstring", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $r := "Hugo Rocks!" | resources.FromString "rocks/hugo.txt" }} +{{ $r.Content }}|{{ $r.RelPermalink }}|{{ $r.Permalink }}|{{ $r.MediaType.Type }} +`) + + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `Hugo Rocks!|/rocks/hugo.txt|http://example.com/rocks/hugo.txt|text/plain`) + b.AssertFileContent("public/rocks/hugo.txt", "Hugo Rocks!") + + }}, + {"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) {}}, + } + + for _, test := range tests { + if !test.shouldRun() { + t.Log("Skip", test.name) + continue + } + + b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger()) + b.WithSimpleConfigFile() + b.WithContent("page.md", ` +--- +title: Hello +--- + +`) + + b.WithSourceFile(filepath.Join("static", "css", "styles1.css"), ` +h1 { + font-style: bold; +} +`) + + b.WithSourceFile(filepath.Join("static", "js", "script1.js"), ` +var x; +x = 5; +document.getElementById("demo").innerHTML = x * 10; +`) + + b.WithSourceFile(filepath.Join("static", "mydata", "json1.json"), ` +{ +"employees":[ + {"firstName":"John", "lastName":"Doe"}, + {"firstName":"Anna", "lastName":"Smith"}, + {"firstName":"Peter", "lastName":"Jones"} +] +} +`) + + b.WithSourceFile(filepath.Join("static", "mydata", "svg1.svg"), ` + + + +`) + + b.WithSourceFile(filepath.Join("static", "mydata", "xml1.xml"), ` + +Hugo Rocks! + +`) + + b.WithSourceFile(filepath.Join("static", "mydata", "html1.html"), ` + + +Cool + + +`) + + b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), ` +$color: #333; + +body { + color: $color; +} +`) + t.Log("Test", test.name) + test.prepare(b) + b.Build(BuildCfg{}) + test.verify(b) + } +} diff --git a/hugolib/site.go b/hugolib/site.go index 83121677990..5e545e3ab73 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -27,12 +27,12 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/resource" + "github.com/gohugoio/hugo/langs" src "github.com/gohugoio/hugo/source" - "github.com/gohugoio/hugo/resource" - "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/config" @@ -140,8 +140,7 @@ type Site struct { renderFormats output.Formats // Logger etc. - *deps.Deps `json:"-"` - resourceSpec *resource.Spec + *deps.Deps `json:"-"` // The func used to title case titles. titleFunc func(s string) string @@ -188,7 +187,6 @@ func (s *Site) reset() *Site { outputFormatsConfig: s.outputFormatsConfig, frontmatterHandler: s.frontmatterHandler, mediaTypesConfig: s.mediaTypesConfig, - resourceSpec: s.resourceSpec, Language: s.Language, owner: s.owner, PageCollections: newPageCollections()} @@ -691,7 +689,11 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { logger = helpers.NewDistinctFeedbackLogger() ) - for _, ev := range events { + cachePartitions := make([]string, len(events)) + + for i, ev := range events { + cachePartitions[i] = resource.ResourceKeyPartition(ev.Name) + if s.isContentDirEvent(ev) { logger.Println("Source changed", ev) sourceChanged = append(sourceChanged, ev) @@ -717,6 +719,9 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { } } + // These in memory resource caches will be rebuilt on demand. + s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) + if len(tmplChanged) > 0 || len(i18nChanged) > 0 { sites := s.owner.Sites first := sites[0] @@ -731,7 +736,11 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { for i := 1; i < len(sites); i++ { site := sites[i] var err error - site.Deps, err = first.Deps.ForLanguage(site.Language) + depsCfg := deps.DepsCfg{ + Language: site.Language, + MediaTypes: site.mediaTypesConfig, + } + site.Deps, err = first.Deps.ForLanguage(depsCfg) if err != nil { return whatChanged{}, err } @@ -797,6 +806,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { if err := s.readAndProcessContent(filenamesChanged...); err != nil { return whatChanged{}, err } + } changed := whatChanged{ @@ -1240,7 +1250,7 @@ func (s *Site) readAndProcessContent(filenames ...string) error { mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor} - sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) if s.running() { // Need to track changes. @@ -1717,6 +1727,8 @@ func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts templName = templ.Name() } s.DistinctErrorLog.Printf("Failed to render %q: %s", templName, r) + s.DistinctErrorLog.Printf("Stack Trace:\n%s", stackTrace(1200)) + // TOD(bep) we really need to fix this. Also see below. if !s.running() && !testMode { os.Exit(-1) diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 93ea5032e2f..9fe60c43466 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -441,7 +441,7 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { content := readDestination(s.T, s.Fs, filename) for _, match := range matches { if !strings.Contains(content, match) { - s.Fatalf("No match for %q in content for %s\n%s", match, filename, content) + s.Fatalf("No match for %q in content for %s\n%s\n%q", match, filename, content, content) } } } @@ -519,7 +519,7 @@ func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *helpers.PathSpec { return ps } -func newTestDefaultPathSpec() *helpers.PathSpec { +func newTestDefaultPathSpec(t *testing.T) *helpers.PathSpec { v := viper.New() // Easier to reason about in tests. v.Set("disablePathToLower", true) @@ -528,8 +528,14 @@ func newTestDefaultPathSpec() *helpers.PathSpec { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") fs := hugofs.NewDefault(v) - ps, _ := helpers.NewPathSpec(fs, v) + ps, err := helpers.NewPathSpec(fs, v) + if err != nil { + t.Fatal(err) + } return ps } diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index c5c962c1630..5075839ff2f 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -205,6 +205,9 @@ func TestI18nTranslate(t *testing.T) { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") // Test without and with placeholders for _, enablePlaceholders := range []bool{false, true} { diff --git a/magefile.go b/magefile.go index 0cede2697b6..b71a07a784a 100644 --- a/magefile.go +++ b/magefile.go @@ -46,17 +46,18 @@ func Vendor() error { // Build hugo binary func Hugo() error { - return sh.RunWith(flagEnv(), goexe, "build", "-ldflags", ldflags, packageName) + return sh.RunWith(flagEnv(), goexe, "build", "-ldflags", ldflags, "-tags", buildTags(), packageName) } // Build hugo binary with race detector enabled func HugoRace() error { - return sh.RunWith(flagEnv(), goexe, "build", "-race", "-ldflags", ldflags, packageName) + return sh.RunWith(flagEnv(), goexe, "build", "-race", "-ldflags", ldflags, "-tags", buildTags(), packageName) } // Install hugo binary +// TODO(bep) resource build tags. func Install() error { - return sh.RunWith(flagEnv(), goexe, "install", "-ldflags", ldflags, packageName) + return sh.RunWith(flagEnv(), goexe, "install", "-ldflags", ldflags, "-tags", buildTags(), packageName) } func flagEnv() map[string]string { @@ -111,18 +112,19 @@ func Check() { } // Run tests in 32-bit mode +// Note that we don't run with the extended tag. Currently not supported in 32 bit. func Test386() error { return sh.RunWith(map[string]string{"GOARCH": "386"}, goexe, "test", "./...") } // Run tests func Test() error { - return sh.Run(goexe, "test", "./...") + return sh.Run(goexe, "test", "./...", "-tags", buildTags()) } // Run tests with race detector func TestRace() error { - return sh.Run(goexe, "test", "-race", "./...") + return sh.Run(goexe, "test", "-race", "./...", "-tags", buildTags()) } // Run gofmt linter @@ -266,3 +268,10 @@ func CheckVendor() error { func isGoLatest() bool { return strings.Contains(runtime.Version(), "1.10") } + +func buildTags() string { + if runtime.GOOS == "windows" { + return "none" + } + return "extended" +} diff --git a/media/mediaType.go b/media/mediaType.go index 33ccb281852..d276f072db5 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -85,24 +85,34 @@ func (m Type) FullSuffix() string { var ( CalendarType = Type{"text", "calendar", "ics", defaultDelimiter} CSSType = Type{"text", "css", "css", defaultDelimiter} + SCSSType = Type{"text", "x-scss", "scss", defaultDelimiter} CSVType = Type{"text", "csv", "csv", defaultDelimiter} HTMLType = Type{"text", "html", "html", defaultDelimiter} JavascriptType = Type{"application", "javascript", "js", defaultDelimiter} JSONType = Type{"application", "json", "json", defaultDelimiter} RSSType = Type{"application", "rss", "xml", defaultDelimiter} XMLType = Type{"application", "xml", "xml", defaultDelimiter} - TextType = Type{"text", "plain", "txt", defaultDelimiter} + // The official MIME type of SVG is image/svg+xml. We currently only support one extension + // per mime type. The workaround in projects is to create multiple media type definitions, + // but we need to improve this to take other known suffixes into account. + // But until then, svg has an svg extension, which is very common. TODO(bep) + SVGType = Type{"image", "svg", "svg", defaultDelimiter} + TextType = Type{"text", "plain", "txt", defaultDelimiter} + + OctetType = Type{"application", "octet-stream", "", ""} ) var DefaultTypes = Types{ CalendarType, CSSType, CSVType, + SCSSType, HTMLType, JavascriptType, JSONType, RSSType, XMLType, + SVGType, TextType, } diff --git a/media/mediaType_test.go b/media/mediaType_test.go index 0cdecdeb11c..8949102c384 100644 --- a/media/mediaType_test.go +++ b/media/mediaType_test.go @@ -30,12 +30,15 @@ func TestDefaultTypes(t *testing.T) { }{ {CalendarType, "text", "calendar", "ics", "text/calendar", "text/calendar+ics"}, {CSSType, "text", "css", "css", "text/css", "text/css+css"}, + {SCSSType, "text", "x-scss", "scss", "text/x-scss", "text/x-scss+scss"}, {CSVType, "text", "csv", "csv", "text/csv", "text/csv+csv"}, {HTMLType, "text", "html", "html", "text/html", "text/html+html"}, {JavascriptType, "application", "javascript", "js", "application/javascript", "application/javascript+js"}, {JSONType, "application", "json", "json", "application/json", "application/json+json"}, {RSSType, "application", "rss", "xml", "application/rss", "application/rss+xml"}, + {SVGType, "image", "svg", "svg", "image/svg", "image/svg+svg"}, {TextType, "text", "plain", "txt", "text/plain", "text/plain+txt"}, + {XMLType, "application", "xml", "xml", "application/xml", "application/xml+xml"}, } { require.Equal(t, test.expectedMainType, test.tp.MainType) require.Equal(t, test.expectedSubType, test.tp.SubType) diff --git a/resource/bundler/bundler.go b/resource/bundler/bundler.go new file mode 100644 index 00000000000..7dd279ad0ff --- /dev/null +++ b/resource/bundler/bundler.go @@ -0,0 +1,122 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bundler contains functions for concatenation etc. of Resource objects. +package bundler + +import ( + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resource" +) + +// Client contains methods perform concatenation and other bundling related +// tasks to Resource objects. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type multiReadSeekCloser struct { + mr io.Reader + sources []resource.ReadSeekCloser +} + +func (r *multiReadSeekCloser) Read(p []byte) (n int, err error) { + return r.mr.Read(p) +} + +func (r *multiReadSeekCloser) Seek(offset int64, whence int) (newOffset int64, err error) { + for _, s := range r.sources { + newOffset, err = s.Seek(offset, whence) + if err != nil { + return + } + } + return +} + +func (r *multiReadSeekCloser) Close() error { + for _, s := range r.sources { + s.Close() + } + return nil +} + +// ConcatTo concatenates the list of Resource objects. +func (c *Client) ConcatTo(targetPath string, resources []resource.Resource) (resource.Resource, error) { + // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. + return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { + var resolvedm media.Type + + // The given set of resources must be of the same Media Type. + // We may improve on that in the future, but then we need to know more. + for i, r := range resources { + if i > 0 && r.MediaType() != resolvedm { + return nil, errors.New("resources in Concat must be of the same Media Type") + } + resolvedm = r.MediaType() + } + + concatr := func() (resource.ReadSeekCloser, error) { + var rcsources []resource.ReadSeekCloser + for _, s := range resources { + rcr, ok := s.(resource.ReadSeekCloserResource) + if !ok { + return nil, fmt.Errorf("resource %T does not implement resource.ReadSeekerCloserResource", s) + } + rc, err := rcr.ReadSeekCloser() + if err != nil { + // Close the already opened. + for _, rcs := range rcsources { + rcs.Close() + } + } + if err != nil { + return nil, err + } + rcsources = append(rcsources, rc) + } + + readers := make([]io.Reader, len(rcsources)) + for i := 0; i < len(rcsources); i++ { + readers[i] = rcsources[i] + } + + mr := io.MultiReader(readers...) + + return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil + } + + composite, err := c.rs.NewForFs( + c.rs.BaseFs.Resources.Fs, + resource.ResourceSourceDescriptor{ + OpenReadSeekCloser: concatr, + RelTargetFilename: filepath.Clean(targetPath)}) + + if err != nil { + return nil, err + } + + return composite, composite.(resource.Source).Publish() + }) + +} diff --git a/resource/create/create.go b/resource/create/create.go new file mode 100644 index 00000000000..f545123bfe3 --- /dev/null +++ b/resource/create/create.go @@ -0,0 +1,115 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package create contains functions for to create Resource objects. This will +// typically non-files. +package create + +import ( + "io" + "path/filepath" + + "github.com/spf13/afero" + + "github.com/dsnet/golib/memfile" + "github.com/gohugoio/hugo/resource" +) + +// Client contains methods to create Resource objects. +// tasks to Resource objects. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type memFileCloser struct { + *memfile.File + io.Closer +} + +func (m *memFileCloser) Close() error { + return nil +} + +// Open creates a new Resource by opening the given filename in the given filesystem. +func (c *Client) Open(fs afero.Fs, filename string) (resource.Resource, error) { + filename = filepath.Clean(filename) + return c.rs.ResourceCache.GetOrCreate(resource.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) { + r, err := c.rs.NewForFs(fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + SourceFilename: filename}) + + if err != nil { + return nil, err + } + + return r, nil + + }) + +} + +// FromString creates a new Resource from a string with the given relative target path. +func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { + r, err := c.rs.NewForFs( + c.rs.BaseFs.Resources.Fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (resource.ReadSeekCloser, error) { + return &memFileCloser{ + File: memfile.New([]byte(content)), + }, nil + }, + RelTargetFilename: filepath.Clean(targetPath)}) + + if err != nil { + return nil, err + } + + return r, nil + + }) + +} + +// FromTemplate creates a new Resource from the given template that will be parsed +// and executed with the given data as context. +// TODO(bep) implement +func (c *Client) FromTemplate(targetPath, templ string, data interface{}) (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { + r, err := c.rs.NewForFs( + c.rs.BaseFs.Resources.Fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (resource.ReadSeekCloser, error) { + return &memFileCloser{ + File: memfile.New([]byte(templ)), + }, nil + }, + RelTargetFilename: filepath.Clean(targetPath)}) + + if err != nil { + return nil, err + } + + return r, nil + + }) + +} diff --git a/resource/image.go b/resource/image.go index 19b68a2966d..6aa382331a9 100644 --- a/resource/image.go +++ b/resource/image.go @@ -19,14 +19,12 @@ import ( "image/color" "io" "os" - "path/filepath" "strconv" "strings" "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/afero" // Importing image codecs for image.DecodeConfig "image" @@ -132,8 +130,6 @@ type Image struct { format imaging.Format - hash string - *genericResource } @@ -151,7 +147,6 @@ func (i *Image) Height() int { func (i *Image) WithNewBase(base string) Resource { return &Image{ imaging: i.imaging, - hash: i.hash, format: i.format, genericResource: i.genericResource.WithNewBase(base).(*genericResource)} } @@ -209,7 +204,7 @@ type imageConfig struct { } func (i *Image) isJPEG() bool { - name := strings.ToLower(i.relTargetPath.file) + name := strings.ToLower(i.relTargetDirFile.file) return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg") } @@ -241,7 +236,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci := i.clone() errOp := action - errPath := i.AbsSourceFilename() + errPath := i.sourceFilename ci.setBasePath(conf) @@ -273,7 +268,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} ci.configLoaded = true - return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.target()) + return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.targetFilename()) }) } @@ -415,11 +410,11 @@ func (i *Image) initConfig() error { } var ( - f afero.File + f ReadSeekCloser config image.Config ) - f, err = i.sourceFs().Open(i.AbsSourceFilename()) + f, err = i.ReadSeekCloser() if err != nil { return } @@ -440,19 +435,19 @@ func (i *Image) initConfig() error { } func (i *Image) decodeSource() (image.Image, error) { - file, err := i.sourceFs().Open(i.AbsSourceFilename()) + f, err := i.ReadSeekCloser() if err != nil { return nil, fmt.Errorf("failed to open image for decode: %s", err) } - defer file.Close() - img, _, err := image.Decode(file) + defer f.Close() + img, _, err := image.Decode(f) return img, err } func (i *Image) copyToDestination(src string) error { var res error i.copyToDestinationInit.Do(func() { - target := i.target() + target := i.targetFilename() // Fast path: // This is a processed version of the original. @@ -469,20 +464,9 @@ func (i *Image) copyToDestination(src string) error { } defer in.Close() - out, err := i.spec.BaseFs.PublishFs.Create(target) - if err != nil && os.IsNotExist(err) { - // When called from shortcodes, the target directory may not exist yet. - // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { - res = err - return - } - out, err = i.spec.BaseFs.PublishFs.Create(target) - if err != nil { - res = err - return - } - } else if err != nil { + out, err := openFileForWriting(i.spec.BaseFs.PublishFs, target) + + if err != nil { res = err return } @@ -501,21 +485,10 @@ func (i *Image) copyToDestination(src string) error { return nil } -func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, filename string) error { - target := filepath.Clean(filename) +func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, targetFilename string) error { - file1, err := i.spec.BaseFs.PublishFs.Create(target) - if err != nil && os.IsNotExist(err) { - // When called from shortcodes, the target directory may not exist yet. - // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { - return err - } - file1, err = i.spec.BaseFs.PublishFs.Create(target) - if err != nil { - return err - } - } else if err != nil { + file1, err := openFileForWriting(i.spec.BaseFs.PublishFs, targetFilename) + if err != nil { return err } @@ -525,11 +498,7 @@ func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resource if resourceCacheFilename != "" { // Also save it to the image resource cache for later reuse. - if err = i.spec.BaseFs.ResourcesFs.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil { - return err - } - - file2, err := i.spec.BaseFs.ResourcesFs.Create(resourceCacheFilename) + file2, err := openFileForWriting(i.spec.BaseFs.Resources.Fs, resourceCacheFilename) if err != nil { return err } @@ -572,17 +541,16 @@ func (i *Image) clone() *Image { return &Image{ imaging: i.imaging, - hash: i.hash, format: i.format, genericResource: &g} } func (i *Image) setBasePath(conf imageConfig) { - i.relTargetPath = i.relTargetPathFromConfig(conf) + i.relTargetDirFile = i.relTargetPathFromConfig(conf) } func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { - p1, p2 := helpers.FileAndExt(i.relTargetPath.file) + p1, p2 := helpers.FileAndExt(i.relTargetDirFile.file) idStr := fmt.Sprintf("_hu%s_%d", i.hash, i.osFileInfo.Size()) @@ -611,7 +579,7 @@ func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { } return dirFile{ - dir: i.relTargetPath.dir, + dir: i.relTargetDirFile.dir, file: fmt.Sprintf("%s%s_%s%s", p1, idStr, key, p2), } diff --git a/resource/image_cache.go b/resource/image_cache.go index 5985797d6b7..4fb45c17f00 100644 --- a/resource/image_cache.go +++ b/resource/image_cache.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -60,12 +60,6 @@ func (c *imageCache) getOrCreate( relTarget := parent.relTargetPathFromConfig(conf) key := parent.relTargetPathForRel(relTarget.path(), false) - if c.pathSpec.Language != nil { - // Avoid do and store more work than needed. The language versions will in - // most cases be duplicates of the same image files. - key = strings.TrimPrefix(key, "/"+c.pathSpec.Language.Lang) - } - // First check the in-memory store, then the disk. c.mu.RLock() img, found := c.store[key] @@ -88,17 +82,17 @@ func (c *imageCache) getOrCreate( // but the count of processed image variations for this site. c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.ResourcesFs) + exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.Resources.Fs) if err != nil { return nil, err } if exists { img = parent.clone() - img.relTargetPath.file = relTarget.file + img.relTargetDirFile.file = relTarget.file img.sourceFilename = cacheFilename - // We have to look resources file system for this. - img.overriddenSourceFs = img.spec.BaseFs.ResourcesFs + // We have to look in the resources file system for this. + img.overriddenSourceFs = img.spec.BaseFs.Resources.Fs } else { img, err = create(cacheFilename) if err != nil { diff --git a/resource/image_test.go b/resource/image_test.go index 11807d69500..f4d91bd9932 100644 --- a/resource/image_test.go +++ b/resource/image_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -78,19 +78,19 @@ func TestImageTransformBasic(t *testing.T) { assert.NoError(err) assert.Equal(320, resized0x.Width()) assert.Equal(200, resized0x.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized0x.RelPermalink(), 320, 200) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized0x.RelPermalink(), 320, 200) resizedx0, err := image.Resize("200x") assert.NoError(err) assert.Equal(200, resizedx0.Width()) assert.Equal(125, resizedx0.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedx0.RelPermalink(), 200, 125) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedx0.RelPermalink(), 200, 125) resizedAndRotated, err := image.Resize("x200 r90") assert.NoError(err) assert.Equal(125, resizedAndRotated.Width()) assert.Equal(200, resizedAndRotated.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAndRotated.RelPermalink(), 125, 200) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAndRotated.RelPermalink(), 125, 200) assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink()) assert.Equal(300, resized.Width()) @@ -115,20 +115,20 @@ func TestImageTransformBasic(t *testing.T) { assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink()) assert.Equal(200, filled.Width()) assert.Equal(100, filled.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filled.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filled.RelPermalink(), 200, 100) smart, err := image.Fill("200x100 smart") assert.NoError(err) assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink()) assert.Equal(200, smart.Width()) assert.Equal(100, smart.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, smart.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, smart.RelPermalink(), 200, 100) // Check cache filledAgain, err := image.Fill("200x100 bottomLeft") assert.NoError(err) assert.True(filled == filledAgain) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filledAgain.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filledAgain.RelPermalink(), 200, 100) } @@ -298,7 +298,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink()) assert.Equal(101, resized.Width()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized.RelPermalink(), 101, 101) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized.RelPermalink(), 101, 101) publishedImageFilename := filepath.Clean(resized.RelPermalink()) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename)) @@ -310,7 +310,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.NoError(err) assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink()) assert.Equal(101, resizedAgain.Width()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAgain.RelPermalink(), 101, 101) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAgain.RelPermalink(), 101, 101) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) } diff --git a/resource/integrity/integrity.go b/resource/integrity/integrity.go new file mode 100644 index 00000000000..c4333531cde --- /dev/null +++ b/resource/integrity/integrity.go @@ -0,0 +1,69 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integrity + +import ( + "crypto" + "encoding/hex" + "hash" + "io" + + "github.com/gohugoio/hugo/resource" +) + +// Client contains methods to fingerprint (cachebusting) and other integrity-related +// methods. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type fingerprintTransformation struct { +} + +func (t *fingerprintTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("fingerprint") +} + +// Transform creates a MD5 hash of the Resource content and inserts that hash before +// the extension in the filename. +func (t *fingerprintTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + h := crypto.MD5.New() + io.Copy(io.MultiWriter(h, ctx.To), ctx.From) + d, err := digestHash(h) + if err != nil { + return err + } + // TODO(bep) make the digest available as a getter somehow. + ctx.AddOutPathIdentifier("." + d) + return nil +} + +// Fingerprint applies fingerprinting of the given resource. +func (c *Client) Fingerprint(res resource.Resource) (resource.Resource, error) { + return c.rs.Transform( + res, + &fingerprintTransformation{}, + ) +} + +func digestHash(h hash.Hash) (string, error) { + sum := h.Sum(nil) + enc := hex.EncodeToString(sum[:]) + return enc, nil +} diff --git a/resource/integrity/integrity_test.go b/resource/integrity/integrity_test.go new file mode 100644 index 00000000000..602db4e38ad --- /dev/null +++ b/resource/integrity/integrity_test.go @@ -0,0 +1,54 @@ +// Copyright 2018-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integrity + +import ( + "github.com/gohugoio/hugo/media" +) + +type testResource struct { + content string +} + +func (r testResource) Permalink() string { + panic("not implemented") +} + +func (r testResource) RelPermalink() string { + panic("not implemented") +} + +func (r testResource) ResourceType() string { + panic("not implemented") +} + +func (r testResource) Name() string { + panic("not implemented") +} + +func (r testResource) MediaType() media.Type { + panic("not implemented") +} + +func (r testResource) Title() string { + panic("not implemented") +} + +func (r testResource) Params() map[string]interface{} { + panic("not implemented") +} + +func (r testResource) Bytes() ([]byte, error) { + return []byte(r.content), nil +} diff --git a/resource/minifiers/minify.go b/resource/minifiers/minify.go new file mode 100644 index 00000000000..706bb0cd200 --- /dev/null +++ b/resource/minifiers/minify.go @@ -0,0 +1,115 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package minifiers + +import ( + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/resource" + "github.com/tdewolff/minify" + "github.com/tdewolff/minify/css" + "github.com/tdewolff/minify/html" + "github.com/tdewolff/minify/js" + "github.com/tdewolff/minify/json" + "github.com/tdewolff/minify/svg" + "github.com/tdewolff/minify/xml" +) + +// Client for minification of Resource objects. Supported minfiers are: +// css, html, js, json, svg and xml. +type Client struct { + rs *resource.Spec + m *minify.M +} + +// New creates a new Client given a specification. Note that it is the media types +// configured for the site that is used to match files to the correct minifier. +func New(rs *resource.Spec) *Client { + m := minify.New() + mt := rs.MediaTypes + + // We use the Type definition of the media types defined in the site if found. + addMinifierFunc(m, mt, "text/css", "css", css.Minify) + addMinifierFunc(m, mt, "text/html", "html", html.Minify) + addMinifierFunc(m, mt, "application/javascript", "js", js.Minify) + addMinifierFunc(m, mt, "application/json", "json", json.Minify) + addMinifierFunc(m, mt, "image/svg", "xml", svg.Minify) + addMinifierFunc(m, mt, "application/xml", "xml", xml.Minify) + + return &Client{rs: rs, m: m} +} + +func addMinifierFunc(m *minify.M, mt media.Types, typeString, suffix string, fn minify.MinifierFunc) { + resolvedTypeStr := resolveMediaTypeString(mt, typeString, suffix) + m.AddFunc(resolvedTypeStr, fn) + if resolvedTypeStr != typeString { + m.AddFunc(typeString, fn) + } +} + +type minifyTransformation struct { + rs *resource.Spec + m *minify.M +} + +func (t *minifyTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("minify") +} + +func (t *minifyTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + mtype := resolveMediaTypeString( + t.rs.MediaTypes, + ctx.InMediaType.Type(), + helpers.ExtNoDelimiter(ctx.InPath), + ) + + if err := t.m.Minify(mtype, ctx.To, ctx.From); err != nil { + return err + } + ctx.AddOutPathIdentifier(".min") + return nil +} + +func (c *Client) Minify(res resource.Resource) (resource.Resource, error) { + return c.rs.Transform( + res, + &minifyTransformation{ + rs: c.rs, + m: c.m}, + ) +} + +func resolveMediaTypeString(types media.Types, typeStr, suffix string) string { + if m, found := resolveMediaType(types, typeStr, suffix); found { + return m.Type() + } + // Fall back to the default. + return typeStr +} + +// Make sure we match the matching pattern with what the user have actually defined +// in his or hers media types configuration. +func resolveMediaType(types media.Types, typeStr, suffix string) (media.Type, bool) { + if m, found := types.GetByType(typeStr); found { + return m, true + } + + if m, found := types.GetBySuffix(suffix); found { + return m, true + } + + return media.Type{}, false + +} diff --git a/resource/postcss/postcss.go b/resource/postcss/postcss.go new file mode 100644 index 00000000000..7dd27b2f9d6 --- /dev/null +++ b/resource/postcss/postcss.go @@ -0,0 +1,175 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postcss + +import ( + "fmt" + "io" + "path/filepath" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/mitchellh/mapstructure" + // "io/ioutil" + "os" + "os/exec" + + "github.com/gohugoio/hugo/common/errors" + + "github.com/gohugoio/hugo/resource" +) + +// Some of the options from https://github.com/postcss/postcss-cli +type Options struct { + + // Set a custom path to look for a config file. + Config string + + NoMap bool `mapstructure:"no-map"` // Disable the default inline sourcemaps + + // Options for when not using a config file + Use string // List of postcss plugins to use + Parser string // Custom postcss parser + Stringifier string // Custom postcss stringifier + Syntax string // Custom postcss syntax +} + +func DecodeOptions(m map[string]interface{}) (opts Options, err error) { + if m == nil { + return + } + err = mapstructure.WeakDecode(m, &opts) + return +} + +func (opts Options) toArgs() []string { + var args []string + if opts.NoMap { + args = append(args, "--no-map") + } + if opts.Use != "" { + args = append(args, "--use", opts.Use) + } + if opts.Parser != "" { + args = append(args, "--parser", opts.Parser) + } + if opts.Stringifier != "" { + args = append(args, "--stringifier", opts.Stringifier) + } + if opts.Syntax != "" { + args = append(args, "--syntax", opts.Syntax) + } + return args +} + +// Client is the client used to do PostCSS transformations. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type postcssTransformation struct { + options Options + rs *resource.Spec +} + +func (t *postcssTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("postcss", t.options) +} + +// Transform shells out to postcss-cli to do the heavy lifting. +// For this to work, you need some additional tools. To install them globally: +// npm install -g postcss-cli +// npm install -g autoprefixer +func (t *postcssTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + + const binary = "postcss" + + if _, err := exec.LookPath(binary); err != nil { + // This may be on a CI server etc. Will fall back to pre-built assets. + return errors.FeatureNotAvailableErr + } + + var configFile string + logger := t.rs.Logger + + if t.options.Config != "" { + configFile = t.options.Config + } else { + configFile = "postcss.config.js" + } + + configFile = filepath.Clean(configFile) + + // We need an abolute filename to the config file. + if !filepath.IsAbs(configFile) { + // We resolve this against the virtual Work filesystem, to allow + // this config file to live in one of the themes if needed. + fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile) + if err != nil { + if t.options.Config != "" { + // Only fail if the user specificed config file is not found. + return fmt.Errorf("postcss config %q not found: %s", configFile, err) + } + configFile = "" + } else { + configFile = fi.(hugofs.RealFilenameInfo).RealFilename() + } + } + + var cmdArgs []string + + if configFile != "" { + logger.INFO.Println("postcss: use config file", configFile) + cmdArgs = []string{"--config", configFile} + } + + if optArgs := t.options.toArgs(); len(optArgs) > 0 { + cmdArgs = append(cmdArgs, optArgs...) + } + + cmd := exec.Command(binary, cmdArgs...) + + cmd.Stdout = ctx.To + cmd.Stderr = os.Stderr + + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + + go func() { + defer stdin.Close() + io.Copy(stdin, ctx.From) + }() + + err = cmd.Run() + if err != nil { + return err + } + + return nil +} + +// Process transforms the given Resource with the PostCSS processor. +func (c *Client) Process(res resource.Resource, options Options) (resource.Resource, error) { + return c.rs.Transform( + res, + &postcssTransformation{rs: c.rs, options: options}, + ) +} diff --git a/resource/resource.go b/resource/resource.go index 9a3725f8ad3..33250d7c198 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,20 +14,22 @@ package resource import ( + "errors" "fmt" + "io" + "io/ioutil" "mime" "os" "path" "path/filepath" - "strconv" "strings" "sync" - "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/loggers" - "github.com/spf13/afero" + jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/cast" + "github.com/spf13/afero" "github.com/gobwas/glob" "github.com/gohugoio/hugo/helpers" @@ -36,11 +38,14 @@ import ( ) var ( + _ ContentResource = (*genericResource)(nil) + _ ReadSeekCloserResource = (*genericResource)(nil) _ Resource = (*genericResource)(nil) - _ metaAssigner = (*genericResource)(nil) _ Source = (*genericResource)(nil) _ Cloner = (*genericResource)(nil) _ ResourcesLanguageMerger = (*Resources)(nil) + _ permalinker = (*genericResource)(nil) + _ resourceHasher = (*genericResource)(nil) ) const DefaultResourceType = "unknown" @@ -48,22 +53,28 @@ const DefaultResourceType = "unknown" // Source is an internal template and not meant for use in the templates. It // may change without notice. type Source interface { - AbsSourceFilename() string Publish() error } +type resourceHasher interface { + initHashFrom(r ReadSeekCloser) error + initHash() error + getHash() (string, error) +} + +type permalinker interface { + relPermalinkFor(target string) string + permalinkFor(target string) string + relTargetPathFor(target string) string + relTargetPath() string +} + // Cloner is an internal template and not meant for use in the templates. It // may change without notice. type Cloner interface { WithNewBase(base string) Resource } -type metaAssigner interface { - setTitle(title string) - setName(name string) - updateParams(params map[string]interface{}) -} - // Resource represents a linkable resource, i.e. a content page, image etc. type Resource interface { // Permalink represents the absolute link to this resource. @@ -77,6 +88,9 @@ type Resource interface { // For content pages, this value is "page". ResourceType() string + // MediaType is this resource's MIME type. + MediaType() media.Type + // Name is the logical name of this resource. This can be set in the front matter // metadata for this resource. If not set, Hugo will assign a value. // This will in most cases be the base filename. @@ -90,6 +104,24 @@ type Resource interface { // Params set in front matter for this resource. Params() map[string]interface{} +} + +type ResourcesLanguageMerger interface { + MergeByLanguage(other Resources) Resources + // Needed for integration with the tpl package. + MergeByLanguageInterface(other interface{}) (interface{}, error) +} + +type translatedResource interface { + TranslationKey() string +} + +// ContentResource represents a Resource that provides a way to get to its content. +// Most Resource types in Hugo implements this interface, including Page. +// This should be used with care, as it will read the file content into memory, but it +// should be cached as effectively as possible by the implementation. +type ContentResource interface { + Resource // Content returns this resource's content. It will be equivalent to reading the content // that RelPermalink points to in the published folder. @@ -100,14 +132,22 @@ type Resource interface { Content() (interface{}, error) } -type ResourcesLanguageMerger interface { - MergeByLanguage(other Resources) Resources - // Needed for integration with the tpl package. - MergeByLanguageInterface(other interface{}) (interface{}, error) +// ReadSeekCloser is implemented by afero.File. We use this as the common type for +// content in Resource objects, even for strings. +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer } -type translatedResource interface { - TranslationKey() string +// OpenReadSeekeCloser allows setting some other way (than reading from a filesystem) +// to open or create a ReadSeekCloser. +type OpenReadSeekCloser func() (ReadSeekCloser, error) + +// ReadSeekCloserResource is a Resource that supports loading its content. +type ReadSeekCloserResource interface { + Resource + ReadSeekCloser() (ReadSeekCloser, error) } // Resources represents a slice of resources, which can be a mix of different types. @@ -125,44 +165,6 @@ func (r Resources) ByType(tp string) Resources { return filtered } -const prefixDeprecatedMsg = `We have added the more flexible Resources.GetMatch (find one) and Resources.Match (many) to replace the "prefix" methods. - -These matches by a given globbing pattern, e.g. "*.jpg". - -Some examples: - -* To find all resources by its prefix in the root dir of the bundle: .Match image* -* To find one resource by its prefix in the root dir of the bundle: .GetMatch image* -* To find all JPEG images anywhere in the bundle: .Match **.jpg` - -// GetByPrefix gets the first resource matching the given filename prefix, e.g -// "logo" will match logo.png. It returns nil of none found. -// In potential ambiguous situations, combine it with ByType. -func (r Resources) GetByPrefix(prefix string) Resource { - helpers.Deprecated("Resources", "GetByPrefix", prefixDeprecatedMsg, true) - prefix = strings.ToLower(prefix) - for _, resource := range r { - if matchesPrefix(resource, prefix) { - return resource - } - } - return nil -} - -// ByPrefix gets all resources matching the given base filename prefix, e.g -// "logo" will match logo.png. -func (r Resources) ByPrefix(prefix string) Resources { - helpers.Deprecated("Resources", "ByPrefix", prefixDeprecatedMsg, true) - var matches Resources - prefix = strings.ToLower(prefix) - for _, resource := range r { - if matchesPrefix(resource, prefix) { - matches = append(matches, resource) - } - } - return matches -} - // GetMatch finds the first Resource matching the given pattern, or nil if none found. // See Match for a more complete explanation about the rules used. func (r Resources) GetMatch(pattern string) Resource { @@ -204,10 +206,6 @@ func (r Resources) Match(pattern string) Resources { return matches } -func matchesPrefix(r Resource, prefix string) bool { - return strings.HasPrefix(strings.ToLower(r.Name()), prefix) -} - var ( globCache = make(map[string]glob.Glob) globMu sync.RWMutex @@ -268,81 +266,189 @@ func (r1 Resources) MergeByLanguageInterface(in interface{}) (interface{}, error type Spec struct { *helpers.PathSpec - mimeTypes media.Types + MediaTypes media.Types + + Logger *jww.Notepad // Holds default filter settings etc. imaging *Imaging - imageCache *imageCache + imageCache *imageCache + ResourceCache *ResourceCache - GenImagePath string + GenImagePath string + GenAssetsPath string } -func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { +func NewSpec(s *helpers.PathSpec, logger *jww.Notepad, mimeTypes media.Types) (*Spec, error) { imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) if err != nil { return nil, err } - genImagePath := filepath.FromSlash("_gen/images") + if logger == nil { + logger = loggers.NewErrorLogger() + } - return &Spec{PathSpec: s, - GenImagePath: genImagePath, - imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache( + genImagePath := filepath.FromSlash("_gen/images") + // The transformed assets (CSS etc.) + genAssetsPath := filepath.FromSlash("_gen/assets") + + rs := &Spec{PathSpec: s, + Logger: logger, + GenImagePath: genImagePath, + GenAssetsPath: genAssetsPath, + imaging: &imaging, + MediaTypes: mimeTypes, + imageCache: newImageCache( s, // We're going to write a cache pruning routine later, so make it extremely // unlikely that the user shoots him or herself in the foot // and this is set to a value that represents data he/she // cares about. This should be set in stone once released. genImagePath, - )}, nil -} + )} -func (r *Spec) NewResourceFromFile( - targetPathBuilder func(base string) string, - file source.File, relTargetFilename string) (Resource, error) { + rs.ResourceCache = newResourceCache(rs) + + return rs, nil - return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename) } -func (r *Spec) NewResourceFromFilename( - targetPathBuilder func(base string) string, - absSourceFilename, relTargetFilename string) (Resource, error) { +type ResourceSourceDescriptor struct { + // TargetPathBuilder is a callback to create target paths's relative to its owner. + TargetPathBuilder func(base string) string - fi, err := r.sourceFs().Stat(absSourceFilename) - if err != nil { - return nil, err + // Need one of these to load the resource content. + SourceFile source.File + OpenReadSeekCloser OpenReadSeekCloser + + // If OpenReadSeekerCloser is not set, we use this to open the file. + SourceFilename string + + // The relative target filename without any language code. + RelTargetFilename string + + // Any base path prepeneded to the permalink. + // Typically the language code if this resource should be published to its sub-folder. + URLBase string + + // Any base path prepended to the target path. This will also typically be the + // language code, but setting it here means that it should not have any effect on + // the permalink. + TargetPathBase string + + // Delay publishing until either Permalink or RelPermalink is called. Maybe never. + LazyPublish bool +} + +func (r ResourceSourceDescriptor) Filename() string { + if r.SourceFile != nil { + return r.SourceFile.Filename() } - return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename) + return r.SourceFilename } func (r *Spec) sourceFs() afero.Fs { - return r.PathSpec.BaseFs.ContentFs + return r.PathSpec.BaseFs.Content.Fs } -func (r *Spec) newResource( - targetPathBuilder func(base string) string, - absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) { +/* +TODO(bep) resource +general design from rainy fishing trip - var mimeType string - ext := filepath.Ext(relTargetFilename) - m, found := r.mimeTypes.GetBySuffix(strings.TrimPrefix(ext, ".")) - if found { - mimeType = m.SubType - } else { - mimeType = mime.TypeByExtension(ext) - if mimeType == "" { - mimeType = DefaultResourceType - } else { - mimeType = mimeType[:strings.Index(mimeType, "/")] +* Transformers provides key, i.e. name + all config +* Root key is join (filename + fast hash + nested keys) +* Store in RADIX with key__ +* Do not store cache when IsServer +* Images follow their own path, but unify with above +* Add /resources/_gen/.gcctimestamp +* Do GCC when age .gcctimestamp >= now - gccinterval + + + +*/ + +func (r *Spec) New(fd ResourceSourceDescriptor) (Resource, error) { + return r.newResourceForFs(r.sourceFs(), fd) +} + +func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + return r.newResourceForFs(sourceFs, fd) +} + +func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + if fd.OpenReadSeekCloser == nil { + if fd.SourceFile != nil && fd.SourceFilename != "" { + return nil, errors.New("both SourceFile and AbsSourceFilename provided") + } else if fd.SourceFile == nil && fd.SourceFilename == "" { + return nil, errors.New("either SourceFile or AbsSourceFilename must be provided") } } - gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType) + if fd.URLBase == "" { + fd.URLBase = r.GetURLLanguageBasePath() + } + + if fd.TargetPathBase == "" { + fd.TargetPathBase = r.GetTargetLanguageBasePath() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = fd.Filename() + } + + return r.newResource(sourceFs, fd) +} + +func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + var fi os.FileInfo + var sourceFilename string - if mimeType == "image" { - ext := strings.ToLower(helpers.Ext(absSourceFilename)) + if fd.OpenReadSeekCloser != nil { + + } else if fd.SourceFilename != "" { + var err error + fi, err = sourceFs.Stat(fd.SourceFilename) + if err != nil { + return nil, err + } + sourceFilename = fd.SourceFilename + } else { + fi = fd.SourceFile.FileInfo() + sourceFilename = fd.SourceFile.Filename() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = sourceFilename + } + + ext := filepath.Ext(fd.RelTargetFilename) + mimeType, found := r.MediaTypes.GetBySuffix(strings.TrimPrefix(ext, ".")) + + if !found { + mimeStr := mime.TypeByExtension(ext) + if mimeStr != "" { + mimeType, _ = media.FromString(mimeStr) + } + + } + + gr := r.newGenericResourceWithBase( + sourceFs, + fd.LazyPublish, + fd.OpenReadSeekCloser, + fd.URLBase, + fd.TargetPathBase, + fd.TargetPathBuilder, + fi, + sourceFilename, + fd.RelTargetFilename, + mimeType) + + if mimeType.MainType == "image" { + ext := strings.ToLower(helpers.Ext(sourceFilename)) imgFormat, ok := imageFormats[ext] if !ok { @@ -351,26 +457,20 @@ func (r *Spec) newResource( return gr, nil } - f, err := gr.sourceFs().Open(absSourceFilename) - if err != nil { - return nil, fmt.Errorf("failed to open image source file: %s", err) - } - defer f.Close() - - hash, err := helpers.MD5FromFileFast(f) - if err != nil { + if err := gr.initHash(); err != nil { return nil, err } return &Image{ - hash: hash, format: imgFormat, imaging: r.imaging, genericResource: gr}, nil } return gr, nil + } +// TODO(bep) resource func (r *Spec) IsInCache(key string) bool { // This is used for cache pruning. We currently only have images, but we could // imagine expanding on this. @@ -381,6 +481,12 @@ func (r *Spec) DeleteCacheByPrefix(prefix string) { r.imageCache.deleteByPrefix(prefix) } +func (r *Spec) ClearCaches() { + // TODO(bep) resource + r.imageCache.clear() + r.ResourceCache.clear() +} + func (r *Spec) CacheStats() string { r.imageCache.mu.RLock() defer r.imageCache.mu.RUnlock() @@ -410,18 +516,54 @@ func (d dirFile) path() string { return path.Join(d.dir, d.file) } +type resourcePathDescriptor struct { + // The relative target directory and filename. + relTargetDirFile dirFile + + // Callback used to construct a target path relative to its owner. + targetPathBuilder func(rel string) string + + // baseURLDir is the fixed sub-folder for a resource in permalinks. This will typically + // be the language code if we publish to the language's sub-folder. + baseURLDir string + + // This will normally be the same as above, but this will only apply to publishing + // of resources. + baseTargetPathDir string + + // baseOffset is set when the output format's path has a offset, e.g. for AMP. + baseOffset string +} + type resourceContent struct { content string contentInit sync.Once } +type resourceHash struct { + hash string + hashInit sync.Once +} + +type publishOnce struct { + publisherInit sync.Once + publisherErr error + logger *jww.Notepad +} + +func (l *publishOnce) publish(s Source) error { + l.publisherInit.Do(func() { + l.publisherErr = s.Publish() + if l.publisherErr != nil { + l.logger.ERROR.Printf("failed to publish Resource: %s") + } + }) + return l.publisherErr +} + // genericResource represents a generic linkable resource. type genericResource struct { - // The relative path to this resource. - relTargetPath dirFile - - // Base is set when the output format's path has a offset, e.g. for AMP. - base string + resourcePathDescriptor title string name string @@ -433,6 +575,12 @@ type genericResource struct { // the path to the file on the real filesystem. sourceFilename string + // Will be set if this resource is backed by something other than a file. + openReadSeekerCloser OpenReadSeekCloser + + // A hash of the source content. Is only calculated in caching situations. + *resourceHash + // This may be set to tell us to look in another filesystem for this resource. // We, by default, use the sourceFs filesystem in the spec below. overriddenSourceFs afero.Fs @@ -440,20 +588,104 @@ type genericResource struct { spec *Spec resourceType string - osFileInfo os.FileInfo + mediaType media.Type - targetPathBuilder func(rel string) string + osFileInfo os.FileInfo // We create copies of this struct, so this needs to be a pointer. *resourceContent + + // May be set to signal lazy/delayed publishing. + *publishOnce } func (l *genericResource) Content() (interface{}, error) { + if err := l.initContent(); err != nil { + return nil, err + } + + return l.content, nil +} + +func (l *genericResource) ReadSeekCloser() (ReadSeekCloser, error) { + if l.openReadSeekerCloser != nil { + return l.openReadSeekerCloser() + } + f, err := l.sourceFs().Open(l.sourceFilename) + if err != nil { + return nil, err + } + return f, nil + +} + +func (l *genericResource) MediaType() media.Type { + return l.mediaType +} + +// Implement the Cloner interface. +func (l genericResource) WithNewBase(base string) Resource { + l.baseOffset = base + l.resourceContent = &resourceContent{} + return &l +} + +func (l *genericResource) initHash() error { + var err error + l.hashInit.Do(func() { + var hash string + var f ReadSeekCloser + f, err = l.ReadSeekCloser() + if err != nil { + err = fmt.Errorf("failed to open source file: %s", err) + return + } + defer f.Close() + + hash, err = helpers.MD5FromFileFast(f) + if err != nil { + return + } + l.hash = hash + + }) + + return err +} + +func (l *genericResource) initHashFrom(r ReadSeekCloser) error { + var err error + l.hashInit.Do(func() { + var hash string + hash, err = helpers.MD5FromFileFast(r) + if err != nil { + return + } + l.hash = hash + r.Seek(0, 0) + }) + return err +} + +func (l *genericResource) getHash() (string, error) { + if err := l.initHash(); err != nil { + return "", err + } + return l.hash, nil +} + +func (l *genericResource) initContent() error { var err error l.contentInit.Do(func() { - var b []byte + var r ReadSeekCloser + r, err = l.ReadSeekCloser() + if err != nil { + return + } + defer r.Close() - b, err := afero.ReadFile(l.sourceFs(), l.AbsSourceFilename()) + var b []byte + b, err = ioutil.ReadAll(r) if err != nil { return } @@ -462,7 +694,7 @@ func (l *genericResource) Content() (interface{}, error) { }) - return l.content, err + return err } func (l *genericResource) sourceFs() afero.Fs { @@ -472,12 +704,36 @@ func (l *genericResource) sourceFs() afero.Fs { return l.spec.sourceFs() } +func (l *genericResource) publishIfNeeded() { + if l.publishOnce != nil { + l.publishOnce.publish(l) + } +} + func (l *genericResource) Permalink() string { - return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String()) + l.publishIfNeeded() + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path()), l.spec.BaseURL.HostURL()) } func (l *genericResource) RelPermalink() string { - return l.relPermalinkForRel(l.relTargetPath.path(), true) + l.publishIfNeeded() + return l.relPermalinkFor(l.relTargetDirFile.path()) +} + +func (l *genericResource) relPermalinkFor(target string) string { + return l.relPermalinkForRel(target) + +} +func (l *genericResource) permalinkFor(target string) string { + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target), l.spec.BaseURL.HostURL()) + +} +func (l *genericResource) relTargetPathFor(target string) string { + return l.relTargetPathForRel(target, false) +} + +func (l *genericResource) relTargetPath() string { + return l.relTargetPathForRel(l.targetPath(), false) } func (l *genericResource) Name() string { @@ -514,31 +770,32 @@ func (l *genericResource) updateParams(params map[string]interface{}) { } } -// Implement the Cloner interface. -func (l genericResource) WithNewBase(base string) Resource { - l.base = base - l.resourceContent = &resourceContent{} - return &l +func (l *genericResource) relPermalinkForRel(rel string) string { + return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, true)) } -func (l *genericResource) relPermalinkForRel(rel string, addBasePath bool) string { - return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, addBasePath)) -} - -func (l *genericResource) relTargetPathForRel(rel string, addBasePath bool) string { +func (l *genericResource) relTargetPathForRel(rel string, isURL bool) string { if l.targetPathBuilder != nil { rel = l.targetPathBuilder(rel) } - if l.base != "" { - rel = path.Join(l.base, rel) + if isURL && l.baseURLDir != "" { + rel = path.Join(l.baseURLDir, rel) + } + + if !isURL && l.baseTargetPathDir != "" { + rel = path.Join(l.baseTargetPathDir, rel) + } + + if l.baseOffset != "" { + rel = path.Join(l.baseOffset, rel) } - if addBasePath && l.spec.PathSpec.BasePath != "" { + if isURL && l.spec.PathSpec.BasePath != "" { rel = path.Join(l.spec.PathSpec.BasePath, rel) } - if rel[0] != '/' { + if len(rel) == 0 || rel[0] != '/' { rel = "/" + rel } @@ -549,146 +806,100 @@ func (l *genericResource) ResourceType() string { return l.resourceType } -func (l *genericResource) AbsSourceFilename() string { - return l.sourceFilename -} - func (l *genericResource) String() string { return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) } func (l *genericResource) Publish() error { - f, err := l.sourceFs().Open(l.AbsSourceFilename()) + f, err := l.ReadSeekCloser() if err != nil { return err } defer f.Close() - return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs) + return helpers.WriteToDisk(l.targetFilename(), f, l.spec.BaseFs.PublishFs) } -const counterPlaceHolder = ":counter" - -// AssignMetadata assigns the given metadata to those resources that supports updates -// and matching by wildcard given in `src` using `filepath.Match` with lower cased values. -// This assignment is additive, but the most specific match needs to be first. -// The `name` and `title` metadata field support shell-matched collection it got a match in. -// See https://golang.org/pkg/path/#Match -func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error { - - counters := make(map[string]int) - - for _, r := range resources { - if _, ok := r.(metaAssigner); !ok { - continue - } - - var ( - nameSet, titleSet bool - nameCounter, titleCounter = 0, 0 - nameCounterFound, titleCounterFound bool - resourceSrcKey = strings.ToLower(r.Name()) - ) - - ma := r.(metaAssigner) - for _, meta := range metadata { - src, found := meta["src"] - if !found { - return fmt.Errorf("missing 'src' in metadata for resource") - } - - srcKey := strings.ToLower(cast.ToString(src)) - - glob, err := getGlob(srcKey) - if err != nil { - return fmt.Errorf("failed to match resource with metadata: %s", err) - } - - match := glob.Match(resourceSrcKey) - - if match { - if !nameSet { - name, found := meta["name"] - if found { - name := cast.ToString(name) - if !nameCounterFound { - nameCounterFound = strings.Contains(name, counterPlaceHolder) - } - if nameCounterFound && nameCounter == 0 { - counterKey := "name_" + srcKey - nameCounter = counters[counterKey] + 1 - counters[counterKey] = nameCounter - } - - ma.setName(replaceResourcePlaceholders(name, nameCounter)) - nameSet = true - } - } - - if !titleSet { - title, found := meta["title"] - if found { - title := cast.ToString(title) - if !titleCounterFound { - titleCounterFound = strings.Contains(title, counterPlaceHolder) - } - if titleCounterFound && titleCounter == 0 { - counterKey := "title_" + srcKey - titleCounter = counters[counterKey] + 1 - counters[counterKey] = titleCounter - } - ma.setTitle((replaceResourcePlaceholders(title, titleCounter))) - titleSet = true - } - } - - params, found := meta["params"] - if found { - m := cast.ToStringMap(params) - // Needed for case insensitive fetching of params values - maps.ToLower(m) - ma.updateParams(m) - } - } - } - } - - return nil -} - -func replaceResourcePlaceholders(in string, counter int) string { - return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1) +// Path is stored with Unix style slashes. +func (l *genericResource) targetPath() string { + return l.relTargetDirFile.path() } -func (l *genericResource) target() string { - target := l.relTargetPathForRel(l.relTargetPath.path(), false) - if l.spec.PathSpec.Languages.IsMultihost() { - target = path.Join(l.spec.PathSpec.Language.Lang, target) - } - return filepath.Clean(target) +func (l *genericResource) targetFilename() string { + return filepath.Clean(l.relTargetPath()) } -func (r *Spec) newGenericResource( +// TODO(bep) clean up below +func (r *Spec) newGenericResource(sourceFs afero.Fs, targetPathBuilder func(base string) string, osFileInfo os.FileInfo, sourceFilename, - baseFilename, - resourceType string) *genericResource { + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + false, + nil, + "", + "", + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} + +func (r *Spec) newGenericResourceWithBase( + sourceFs afero.Fs, + lazyPublish bool, + openReadSeekerCloser OpenReadSeekCloser, + urlBaseDir string, + targetPathBaseDir string, + targetPathBuilder func(base string) string, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { // This value is used both to construct URLs and file paths, but start // with a Unix-styled path. baseFilename = filepath.ToSlash(baseFilename) fpath, fname := path.Split(baseFilename) - return &genericResource{ + var resourceType string + if mediaType.MainType == "image" { + resourceType = mediaType.MainType + } else { + resourceType = mediaType.SubType + } + + pathDescriptor := resourcePathDescriptor{ + baseURLDir: urlBaseDir, + baseTargetPathDir: targetPathBaseDir, targetPathBuilder: targetPathBuilder, - osFileInfo: osFileInfo, - sourceFilename: sourceFilename, - relTargetPath: dirFile{dir: fpath, file: fname}, - resourceType: resourceType, - spec: r, - params: make(map[string]interface{}), - name: baseFilename, - title: baseFilename, - resourceContent: &resourceContent{}, + relTargetDirFile: dirFile{dir: fpath, file: fname}, + } + + var po *publishOnce + if lazyPublish { + po = &publishOnce{logger: r.Logger} + } + + return &genericResource{ + openReadSeekerCloser: openReadSeekerCloser, + publishOnce: po, + resourcePathDescriptor: pathDescriptor, + overriddenSourceFs: sourceFs, + osFileInfo: osFileInfo, + sourceFilename: sourceFilename, + mediaType: mediaType, + resourceType: resourceType, + spec: r, + params: make(map[string]interface{}), + name: baseFilename, + title: baseFilename, + resourceContent: &resourceContent{}, + resourceHash: &resourceHash{}, } } diff --git a/resource/resource_cache.go b/resource/resource_cache.go new file mode 100644 index 00000000000..4c4d77635af --- /dev/null +++ b/resource/resource_cache.go @@ -0,0 +1,235 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/spf13/afero" + + "github.com/BurntSushi/locker" +) + +const ( + CACHE_CLEAR_ALL = "clear_all" + CACHE_OTHER = "other" +) + +type ResourceCache struct { + rs *Spec + + cache map[string]Resource + sync.RWMutex + + // Provides named resource locks. + nlocker *locker.Locker +} + +// ResourceKeyPartition returns a partition name +// to allow for more fine grained cache flushes. +// It will return the file extension without the leading ".". If no +// extension, it will return "other". +func ResourceKeyPartition(filename string) string { + ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") + if ext == "" { + ext = CACHE_OTHER + } + return ext +} + +func newResourceCache(rs *Spec) *ResourceCache { + return &ResourceCache{ + rs: rs, + cache: make(map[string]Resource), + nlocker: locker.NewLocker(), + } +} + +func (c *ResourceCache) clear() { + c.Lock() + defer c.Unlock() + + c.cache = make(map[string]Resource) + c.nlocker = locker.NewLocker() +} + +func (c *ResourceCache) get(key string) (Resource, bool) { + c.RLock() + defer c.RUnlock() + r, found := c.cache[key] + return r, found +} + +func (c *ResourceCache) GetOrCreate(partition, key string, f func() (Resource, error)) (Resource, error) { + key = path.Join(partition, key) + // First check in-memory cache. + r, found := c.get(key) + if found { + return r, nil + } + // This is a potentially long running operation, so get a named lock. + c.nlocker.Lock(key) + + // Double check in-memory cache. + r, found = c.get(key) + if found { + c.nlocker.Unlock(key) + return r, nil + } + + defer c.nlocker.Unlock(key) + + r, err := f() + if err != nil { + return nil, err + } + + c.set(key, r) + + return r, nil + +} + +func (c *ResourceCache) getFilenames(key string) (string, string) { + filenameBase := filepath.Join(c.rs.GenAssetsPath, key) + filenameMeta := filenameBase + ".json" + filenameContent := filenameBase + ".content" + + return filenameMeta, filenameContent +} + +func (c *ResourceCache) getFromFile(key string) (afero.File, transformedResourceMetadata, bool) { + c.RLock() + defer c.RUnlock() + + var meta transformedResourceMetadata + filenameMeta, filenameContent := c.getFilenames(key) + fMeta, err := c.rs.Resources.Fs.Open(filenameMeta) + if err != nil { + return nil, meta, false + } + defer fMeta.Close() + + jsonContent, err := ioutil.ReadAll(fMeta) + if err != nil { + return nil, meta, false + } + + if err := json.Unmarshal(jsonContent, &meta); err != nil { + return nil, meta, false + } + + fContent, err := c.rs.Resources.Fs.Open(filenameContent) + if err != nil { + return nil, meta, false + } + + return fContent, meta, true +} + +// writeMeta writes the metadata to file and returns a writer for the content part. +func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) (afero.File, error) { + filenameMeta, filenameContent := c.getFilenames(key) + raw, err := json.Marshal(meta) + if err != nil { + return nil, err + } + + fm, err := c.openResourceFileForWriting(filenameMeta) + if err != nil { + return nil, err + } + + if _, err := fm.Write(raw); err != nil { + return nil, err + } + + return c.openResourceFileForWriting(filenameContent) + +} + +func (c *ResourceCache) openResourceFileForWriting(filename string) (afero.File, error) { + return openFileForWriting(c.rs.Resources.Fs, filename) +} + +// openFileForWriting opens or creates the given file. If the target directory +// does not exist, it gets created. +func openFileForWriting(fs afero.Fs, filename string) (afero.File, error) { + filename = filepath.Clean(filename) + // Create will truncate if file already exists. + f, err := fs.Create(filename) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + if err = fs.MkdirAll(filepath.Dir(filename), 0755); err != nil { + return nil, err + } + f, err = fs.Create(filename) + } + + return f, err +} + +func (c *ResourceCache) set(key string, r Resource) { + c.Lock() + defer c.Unlock() + c.cache[key] = r +} + +func (c *ResourceCache) DeletePartitions(partitions ...string) { + partitionsSet := map[string]bool{ + // Always clear out the resources not matching the partition. + "other": true, + } + for _, p := range partitions { + partitionsSet[p] = true + } + + // TODO(bep) resource send this on "config" changes ... + if partitionsSet[CACHE_CLEAR_ALL] { + c.clear() + return + } + + c.Lock() + defer c.Unlock() + + for k := range c.cache { + clear := false + partIdx := strings.Index(k, "/") + if partIdx == -1 { + clear = true + fmt.Println("CLEAR NO part", k) + } else { + partition := k[:partIdx] + if partitionsSet[partition] { + clear = true + fmt.Println("Clear part", partition) + } + } + + if clear { + delete(c.cache, k) + } + } + +} diff --git a/resource/resource_metadata.go b/resource/resource_metadata.go new file mode 100644 index 00000000000..2c82aeaf642 --- /dev/null +++ b/resource/resource_metadata.go @@ -0,0 +1,129 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "fmt" + "strconv" + + "github.com/spf13/cast" + + "strings" + + "github.com/gohugoio/hugo/common/maps" +) + +var ( + _ metaAssigner = (*genericResource)(nil) +) + +// metaAssigner allows updating metadata in resources that supports it. +type metaAssigner interface { + setTitle(title string) + setName(name string) + updateParams(params map[string]interface{}) +} + +const counterPlaceHolder = ":counter" + +// AssignMetadata assigns the given metadata to those resources that supports updates +// and matching by wildcard given in `src` using `filepath.Match` with lower cased values. +// This assignment is additive, but the most specific match needs to be first. +// The `name` and `title` metadata field support shell-matched collection it got a match in. +// See https://golang.org/pkg/path/#Match +func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error { + + counters := make(map[string]int) + + for _, r := range resources { + if _, ok := r.(metaAssigner); !ok { + continue + } + + var ( + nameSet, titleSet bool + nameCounter, titleCounter = 0, 0 + nameCounterFound, titleCounterFound bool + resourceSrcKey = strings.ToLower(r.Name()) + ) + + ma := r.(metaAssigner) + for _, meta := range metadata { + src, found := meta["src"] + if !found { + return fmt.Errorf("missing 'src' in metadata for resource") + } + + srcKey := strings.ToLower(cast.ToString(src)) + + glob, err := getGlob(srcKey) + if err != nil { + return fmt.Errorf("failed to match resource with metadata: %s", err) + } + + match := glob.Match(resourceSrcKey) + + if match { + if !nameSet { + name, found := meta["name"] + if found { + name := cast.ToString(name) + if !nameCounterFound { + nameCounterFound = strings.Contains(name, counterPlaceHolder) + } + if nameCounterFound && nameCounter == 0 { + counterKey := "name_" + srcKey + nameCounter = counters[counterKey] + 1 + counters[counterKey] = nameCounter + } + + ma.setName(replaceResourcePlaceholders(name, nameCounter)) + nameSet = true + } + } + + if !titleSet { + title, found := meta["title"] + if found { + title := cast.ToString(title) + if !titleCounterFound { + titleCounterFound = strings.Contains(title, counterPlaceHolder) + } + if titleCounterFound && titleCounter == 0 { + counterKey := "title_" + srcKey + titleCounter = counters[counterKey] + 1 + counters[counterKey] = titleCounter + } + ma.setTitle((replaceResourcePlaceholders(title, titleCounter))) + titleSet = true + } + } + + params, found := meta["params"] + if found { + m := cast.ToStringMap(params) + // Needed for case insensitive fetching of params values + maps.ToLower(m) + ma.updateParams(m) + } + } + } + } + + return nil +} + +func replaceResourcePlaceholders(in string, counter int) string { + return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1) +} diff --git a/resource/resource_metadata_test.go b/resource/resource_metadata_test.go new file mode 100644 index 00000000000..85fb25b5756 --- /dev/null +++ b/resource/resource_metadata_test.go @@ -0,0 +1,230 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + "github.com/gohugoio/hugo/media" + + "github.com/stretchr/testify/require" +) + +func TestAssignMetadata(t *testing.T) { + assert := require.New(t) + spec := newTestResourceSpec(assert) + + var foo1, foo2, foo3, logo1, logo2, logo3 Resource + var resources Resources + + for _, this := range []struct { + metaData []map[string]interface{} + assertFunc func(err error) + }{ + {[]map[string]interface{}{ + { + "title": "My Resource", + "name": "My Name", + "src": "*", + }, + }, func(err error) { + assert.Equal("My Resource", logo1.Title()) + assert.Equal("My Name", logo1.Name()) + assert.Equal("My Name", foo2.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "My Logo", + "src": "*loGo*", + }, + { + "title": "My Resource", + "name": "My Name", + "src": "*", + }, + }, func(err error) { + assert.Equal("My Logo", logo1.Title()) + assert.Equal("My Logo", logo2.Title()) + assert.Equal("My Name", logo1.Name()) + assert.Equal("My Name", foo2.Name()) + assert.Equal("My Name", foo3.Name()) + assert.Equal("My Resource", foo3.Title()) + + }}, + {[]map[string]interface{}{ + { + "title": "My Logo", + "src": "*loGo*", + "params": map[string]interface{}{ + "Param1": true, + "icon": "logo", + }, + }, + { + "title": "My Resource", + "src": "*", + "params": map[string]interface{}{ + "Param2": true, + "icon": "resource", + }, + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("My Logo", logo1.Title()) + assert.Equal("My Resource", foo3.Title()) + _, p1 := logo2.Params()["param1"] + _, p2 := foo2.Params()["param2"] + _, p1_2 := foo2.Params()["param1"] + _, p2_2 := logo2.Params()["param2"] + + icon1, _ := logo2.Params()["icon"] + icon2, _ := foo2.Params()["icon"] + + assert.True(p1) + assert.True(p2) + + // Check merge + assert.True(p2_2) + assert.False(p1_2) + + assert.Equal("logo", icon1) + assert.Equal("resource", icon2) + + }}, + {[]map[string]interface{}{ + { + "name": "Logo Name #:counter", + "src": "*logo*", + }, + { + "title": "Resource #:counter", + "name": "Name #:counter", + "src": "*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Resource #2", logo2.Title()) + assert.Equal("Logo Name #1", logo2.Name()) + assert.Equal("Resource #4", logo1.Title()) + assert.Equal("Logo Name #2", logo1.Name()) + assert.Equal("Resource #1", foo2.Title()) + assert.Equal("Resource #3", foo1.Title()) + assert.Equal("Name #2", foo1.Name()) + assert.Equal("Resource #5", foo3.Title()) + + assert.Equal(logo2, resources.GetMatch("logo name #1*")) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo #:counter", + "src": "logo3.png", + }, + { + "title": "Other Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Third Logo #1", logo3.Title()) + assert.Equal("Name #3", logo3.Name()) + assert.Equal("Other Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Other Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo", + "src": "logo3.png", + }, + { + "title": "Other Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Third Logo", logo3.Title()) + assert.Equal("Name #3", logo3.Name()) + assert.Equal("Other Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Other Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "name": "third-logo", + "src": "logo3.png", + }, + { + "title": "Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Logo #3", logo3.Title()) + assert.Equal("third-logo", logo3.Name()) + assert.Equal("Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo #:counter", + }, + }, func(err error) { + // Missing src + assert.Error(err) + + }}, + {[]map[string]interface{}{ + { + "title": "Title", + "src": "[]", + }, + }, func(err error) { + // Invalid pattern + assert.Error(err) + + }}, + } { + + foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) + logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) + foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) + logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) + foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) + logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) + + resources = Resources{ + foo2, + logo2, + foo1, + logo1, + foo3, + logo3, + } + + this.assertFunc(AssignMetadata(this.metaData, resources...)) + } + +} diff --git a/resource/resource_test.go b/resource/resource_test.go index 40061e5c461..659994c364b 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/media" + "github.com/stretchr/testify/require" ) @@ -29,7 +31,7 @@ func TestGenericResource(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) - r := spec.newGenericResource(nil, nil, "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) assert.Equal("https://example.com/foo.css", r.Permalink()) assert.Equal("/foo.css", r.RelPermalink()) @@ -44,7 +46,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := func(s string) string { return path.Join("/foo", s) } - r := spec.newGenericResource(factory, nil, "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) assert.Equal("https://example.com/foo/foo.css", r.Permalink()) assert.Equal("/foo/foo.css", r.RelPermalink()) @@ -58,8 +60,7 @@ func TestNewResourceFromFilename(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") writeSource(t, spec.Fs, "content/a/b/data.json", "json") - r, err := spec.NewResourceFromFilename(nil, - filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"}) assert.NoError(err) assert.NotNil(r) @@ -67,7 +68,7 @@ func TestNewResourceFromFilename(t *testing.T) { assert.Equal("/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/a/b/logo.png", r.Permalink()) - r, err = spec.NewResourceFromFilename(nil, "a/b/data.json", "a/b/data.json") + r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"}) assert.NoError(err) assert.NotNil(r) @@ -84,8 +85,7 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") - r, err := spec.NewResourceFromFilename(nil, - filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")}) assert.NoError(err) assert.NotNil(r) @@ -93,18 +93,20 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { assert.Equal("/docs/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/docs/a/b/logo.png", r.Permalink()) img := r.(*Image) - assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.target()) + assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.targetFilename()) } +var pngType, _ = media.FromString("image/png") + func TestResourcesByType(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo.png", "logo.css", "image"), - spec.newGenericResource(nil, nil, "/a/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/a/foo3.css", "foo3.css", "css")} + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), + spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} assert.Len(resources.ByType("css"), 3) assert.Len(resources.ByType("image"), 1) @@ -115,25 +117,25 @@ func TestResourcesGetByPrefix(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")} - - assert.Nil(resources.GetByPrefix("asdf")) - assert.Equal("/logo1.png", resources.GetByPrefix("logo").RelPermalink()) - assert.Equal("/logo1.png", resources.GetByPrefix("loGo").RelPermalink()) - assert.Equal("/Logo2.png", resources.GetByPrefix("logo2").RelPermalink()) - assert.Equal("/foo2.css", resources.GetByPrefix("foo2").RelPermalink()) - assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink()) - assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink()) - assert.Nil(resources.GetByPrefix("asdfasdf")) - - assert.Equal(2, len(resources.ByPrefix("logo"))) - assert.Equal(1, len(resources.ByPrefix("logo2"))) - - logo := resources.GetByPrefix("logo") + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} + + assert.Nil(resources.GetMatch("asdf*")) + assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink()) + assert.Equal("/logo1.png", resources.GetMatch("loGo*").RelPermalink()) + assert.Equal("/Logo2.png", resources.GetMatch("logo2*").RelPermalink()) + assert.Equal("/foo2.css", resources.GetMatch("foo2*").RelPermalink()) + assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink()) + assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink()) + assert.Nil(resources.GetMatch("asdfasdf*")) + + assert.Equal(2, len(resources.Match("logo*"))) + assert.Equal(1, len(resources.Match("logo2*"))) + + logo := resources.GetMatch("logo*") assert.NotNil(logo.Params()) assert.Equal("logo1.png", logo.Name()) assert.Equal("logo1.png", logo.Title()) @@ -144,14 +146,14 @@ func TestResourcesGetMatch(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/foo4.css", "c/foo4.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/foo5.css", "c/foo5.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", "css"), + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), } assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink()) @@ -186,226 +188,6 @@ func TestResourcesGetMatch(t *testing.T) { } -func TestAssignMetadata(t *testing.T) { - assert := require.New(t) - spec := newTestResourceSpec(assert) - - var foo1, foo2, foo3, logo1, logo2, logo3 Resource - var resources Resources - - for _, this := range []struct { - metaData []map[string]interface{} - assertFunc func(err error) - }{ - {[]map[string]interface{}{ - { - "title": "My Resource", - "name": "My Name", - "src": "*", - }, - }, func(err error) { - assert.Equal("My Resource", logo1.Title()) - assert.Equal("My Name", logo1.Name()) - assert.Equal("My Name", foo2.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "My Logo", - "src": "*loGo*", - }, - { - "title": "My Resource", - "name": "My Name", - "src": "*", - }, - }, func(err error) { - assert.Equal("My Logo", logo1.Title()) - assert.Equal("My Logo", logo2.Title()) - assert.Equal("My Name", logo1.Name()) - assert.Equal("My Name", foo2.Name()) - assert.Equal("My Name", foo3.Name()) - assert.Equal("My Resource", foo3.Title()) - - }}, - {[]map[string]interface{}{ - { - "title": "My Logo", - "src": "*loGo*", - "params": map[string]interface{}{ - "Param1": true, - "icon": "logo", - }, - }, - { - "title": "My Resource", - "src": "*", - "params": map[string]interface{}{ - "Param2": true, - "icon": "resource", - }, - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("My Logo", logo1.Title()) - assert.Equal("My Resource", foo3.Title()) - _, p1 := logo2.Params()["param1"] - _, p2 := foo2.Params()["param2"] - _, p1_2 := foo2.Params()["param1"] - _, p2_2 := logo2.Params()["param2"] - - icon1, _ := logo2.Params()["icon"] - icon2, _ := foo2.Params()["icon"] - - assert.True(p1) - assert.True(p2) - - // Check merge - assert.True(p2_2) - assert.False(p1_2) - - assert.Equal("logo", icon1) - assert.Equal("resource", icon2) - - }}, - {[]map[string]interface{}{ - { - "name": "Logo Name #:counter", - "src": "*logo*", - }, - { - "title": "Resource #:counter", - "name": "Name #:counter", - "src": "*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Resource #2", logo2.Title()) - assert.Equal("Logo Name #1", logo2.Name()) - assert.Equal("Resource #4", logo1.Title()) - assert.Equal("Logo Name #2", logo1.Name()) - assert.Equal("Resource #1", foo2.Title()) - assert.Equal("Resource #3", foo1.Title()) - assert.Equal("Name #2", foo1.Name()) - assert.Equal("Resource #5", foo3.Title()) - - assert.Equal(logo2, resources.GetByPrefix("logo name #1")) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo #:counter", - "src": "logo3.png", - }, - { - "title": "Other Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Third Logo #1", logo3.Title()) - assert.Equal("Name #3", logo3.Name()) - assert.Equal("Other Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Other Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo", - "src": "logo3.png", - }, - { - "title": "Other Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Third Logo", logo3.Title()) - assert.Equal("Name #3", logo3.Name()) - assert.Equal("Other Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Other Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "name": "third-logo", - "src": "logo3.png", - }, - { - "title": "Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Logo #3", logo3.Title()) - assert.Equal("third-logo", logo3.Name()) - assert.Equal("Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo #:counter", - }, - }, func(err error) { - // Missing src - assert.Error(err) - - }}, - {[]map[string]interface{}{ - { - "title": "Title", - "src": "[]", - }, - }, func(err error) { - // Invalid pattern - assert.Error(err) - - }}, - } { - - foo2 = spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css") - logo2 = spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image") - foo1 = spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css") - logo1 = spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image") - foo3 = spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css") - logo3 = spec.newGenericResource(nil, nil, "/b/logo3.png", "logo3.png", "image") - - resources = Resources{ - foo2, - logo2, - foo1, - logo1, - foo3, - logo3, - } - - this.assertFunc(AssignMetadata(this.metaData, resources...)) - } - -} - -func BenchmarkResourcesByPrefix(b *testing.B) { - resources := benchResources(b) - prefixes := []string{"abc", "jkl", "nomatch", "sub/"} - rnd := rand.New(rand.NewSource(time.Now().Unix())) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - resources.ByPrefix(prefixes[rnd.Intn(len(prefixes))]) - } - }) -} - func BenchmarkResourcesMatch(b *testing.B) { resources := benchResources(b) prefixes := []string{"abc*", "jkl*", "nomatch*", "sub/*"} @@ -428,7 +210,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := Resources{spec.newGenericResource(nil, nil, "/a/"+a100, a100, "css")} + resources := Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -444,17 +226,17 @@ func benchResources(b *testing.B) Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/b/sub/"+name, "sub/"+name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) } return resources @@ -482,7 +264,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } b.StartTimer() diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go index 360adc038ab..e78a536a259 100644 --- a/resource/testhelpers_test.go +++ b/resource/testhelpers_test.go @@ -33,7 +33,9 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * cfg.Set("dataDir", "data") cfg.Set("i18nDir", "i18n") cfg.Set("layoutDir", "layouts") + cfg.Set("assetDir", "assets") cfg.Set("archetypeDir", "archetypes") + cfg.Set("publishDir", "public") imagingCfg := map[string]interface{}{ "resampleFilter": "linear", @@ -49,7 +51,7 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * assert.NoError(err) - spec, err := NewSpec(s, media.DefaultTypes) + spec, err := NewSpec(s, nil, media.DefaultTypes) assert.NoError(err) return spec } @@ -72,7 +74,9 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { cfg.Set("dataDir", "data") cfg.Set("i18nDir", "i18n") cfg.Set("layoutDir", "layouts") + cfg.Set("assetDir", "assets") cfg.Set("archetypeDir", "archetypes") + cfg.Set("publishDir", "public") fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} @@ -81,7 +85,7 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { assert.NoError(err) - spec, err := NewSpec(s, media.DefaultTypes) + spec, err := NewSpec(s, nil, media.DefaultTypes) assert.NoError(err) return spec @@ -102,12 +106,11 @@ func fetchImageForSpec(spec *Spec, assert *require.Assertions, name string) *Ima return r.(*Image) } -func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) Resource { +func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) ContentResource { src, err := os.Open(filepath.FromSlash("testdata/" + name)) assert.NoError(err) - assert.NoError(spec.BaseFs.ContentFs.MkdirAll(filepath.Dir(name), 0755)) - out, err := spec.BaseFs.ContentFs.Create(name) + out, err := openFileForWriting(spec.BaseFs.Content.Fs, name) assert.NoError(err) _, err = io.Copy(out, src) out.Close() @@ -118,10 +121,10 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R return path.Join("/a", s) } - r, err := spec.NewResourceFromFilename(factory, name, name) + r, err := spec.New(ResourceSourceDescriptor{TargetPathBuilder: factory, SourceFilename: name}) assert.NoError(err) - return r + return r.(ContentResource) } func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { diff --git a/resource/tocss/scss/client.go b/resource/tocss/scss/client.go new file mode 100644 index 00000000000..085a4ab1f7c --- /dev/null +++ b/resource/tocss/scss/client.go @@ -0,0 +1,119 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scss + +import ( + "github.com/bep/go-tocss/scss" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/resource" + "github.com/mitchellh/mapstructure" +) + +type Client struct { + rs *resource.Spec + sfs *filesystems.SourceFilesystem +} + +func New(fs *filesystems.SourceFilesystem, rs *resource.Spec) (*Client, error) { + return &Client{sfs: fs, rs: rs}, nil +} + +type Options struct { + // Default is nested. + // One of nested, expanded, compact, compressed. + OutputStyle string + + // Precision of floating point math. + Precision int + + // When enabled, Hugo will generate a source map. + EnableSourceMap bool +} + +type options struct { + // The options we receive from the end user. + from Options + + // The options we send to the SCSS library. + to scss.Options +} + +// Options is a local copy of the options supported by go-tocss. We create +// a 64 bit hash of this to detect configuration changes, and need to exclude +// some fields from that calculation. +type _old struct { + // Default is nested. + OutputStyle scss.OutputStyle + + // Precision of floating point math. + Precision int + + // File paths to use to resolve imports. + IncludePaths []string `hash:"ignore"` + + // ImportResolver can be used to supply a custom import resolver, both to redirect + // to another URL or to return the body. + ImportResolver func(url string, prev string) (newURL string, body string, resolved bool) `hash:"ignore"` + + // Source map settings + SourceMapFilename string + SourceMapRoot string + InputPath string + OutputPath string + SourceMapContents bool + OmitSourceMapURL bool + EnableEmbeddedSourceMap bool +} + +func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource, error) { + internalOptions := options{ + from: opts, + } + + // Transfer values from client. + internalOptions.to.Precision = opts.Precision + internalOptions.to.OutputStyle = scss.OutputStyleFromString(opts.OutputStyle) + + // We may allow the end user to add IncludePaths later, if we find a use + // case for that. + internalOptions.to.IncludePaths = c.sfs.RealDirs("scss") // TODO(bep) resource + + if internalOptions.to.Precision == 0 { + // bootstrap-sass requires 8 digits precision. The libsass default is 5. + // https://github.com/twbs/bootstrap-sass/blob/master/README.md#sass-number-precision + internalOptions.to.Precision = 8 + } + + return c.rs.Transform( + res, + &toCSSTransformation{c: c, options: internalOptions}, + ) +} + +type toCSSTransformation struct { + c *Client + options options +} + +func (t *toCSSTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("tocss", t.options) +} + +func DecodeOptions(m map[string]interface{}) (opts Options, err error) { + if m == nil { + return + } + err = mapstructure.WeakDecode(m, &opts) + return +} diff --git a/resource/tocss/scss/tocss.go b/resource/tocss/scss/tocss.go new file mode 100644 index 00000000000..69de8ef4dd7 --- /dev/null +++ b/resource/tocss/scss/tocss.go @@ -0,0 +1,113 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build extended + +package scss + +import ( + "fmt" + "io" + "path" + "strings" + + "github.com/bep/go-tocss/scss" + "github.com/bep/go-tocss/scss/libsass" + "github.com/bep/go-tocss/tocss" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resource" +) + +// Used in tests. This feature requires Hugo to be built with the extended tag. +func Supports() bool { + return true +} + +/* + +TODO(bep) resource /resources/_gen + +* When IsServer: generate to /tmp/somefolder/resources/_gen and add that to + composite. Make sure to get order of composite correct +* Delete tmp on server shutdown +* When !IsServer: --gc, interval? + +*/ +func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + ctx.ReplaceOutPathExtension(".css") + ctx.OutMediaType = media.CSSType + + outName := path.Base(ctx.OutPath) + + // TODO(bep) resource source maps for non-files? Template transform etc. Not sure that + // it matters. + + options := t.options + + if options.from.EnableSourceMap { + + // TODO(bep) resource + options.to.SourceMapFilename = outName + ".map" + options.to.SourceMapRoot = t.c.rs.WorkingDir + + // Setting this to the relative input filename will get the source map + // more correct for the main entry path (main.scss typically), but + // it will mess up the import mappings. As a workaround, we do a replacement + // in the source map itself (see below). + //options.InputPath = inputPath + options.to.OutputPath = outName + options.to.SourceMapContents = true + options.to.OmitSourceMapURL = false + options.to.EnableEmbeddedSourceMap = false + } + + res, err := t.c.toCSS(options.to, ctx.To, ctx.From) + if err != nil { + return err + } + + if options.from.EnableSourceMap && res.SourceMapContent != "" { + // TODO(bep) resource InPath and OutPath should be relative from the start. + // TODO(bep) resource handle the case where this isn't a file. + inputPath := t.c.sfs.RealFilename(strings.TrimPrefix(ctx.InPath, "/")) + if strings.HasPrefix(inputPath, t.c.rs.WorkingDir) { + inputPath = strings.TrimPrefix(inputPath, t.c.rs.WorkingDir+helpers.FilePathSeparator) + } + // This is a workaround for what looks like a bug in Libsass. But + // getting this resolution correct in tools like Chrome Workspaces + // is important enough to go this extra mile. + // TODO(bep) resource create issue in libsass repo + mapContent := strings.Replace(res.SourceMapContent, `"stdin",`, fmt.Sprintf("%q,", inputPath), 1) + return ctx.PublishSourceMap(mapContent) + } + return nil +} + +// TODO(bep) resource +// Options, source maps, ... ? +func (c *Client) toCSS(options scss.Options, dst io.Writer, src io.Reader) (tocss.Result, error) { + var res tocss.Result + + transpiler, err := libsass.New(options) + if err != nil { + return res, err + } + + res, err = transpiler.Execute(dst, src) + if err != nil { + return res, fmt.Errorf("SCSS processing failed: %s", err) + } + + return res, nil +} diff --git a/resource/tocss/scss/tocss_notavailable.go b/resource/tocss/scss/tocss_notavailable.go new file mode 100644 index 00000000000..69b4fc6556e --- /dev/null +++ b/resource/tocss/scss/tocss_notavailable.go @@ -0,0 +1,30 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !extended + +package scss + +import ( + "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/resource" +) + +// Used in tests. +func Supports() bool { + return false +} + +func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + return errors.FeatureNotAvailableErr +} diff --git a/resource/transform.go b/resource/transform.go new file mode 100644 index 00000000000..c4ed4c13bb9 --- /dev/null +++ b/resource/transform.go @@ -0,0 +1,474 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "bytes" + "path" + "strconv" + + "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/helpers" + "github.com/mitchellh/hashstructure" + "github.com/spf13/afero" + + "fmt" + "io" + "sync" + + "github.com/gohugoio/hugo/media" + + bp "github.com/gohugoio/hugo/bufferpool" +) + +var ( + _ ContentResource = (*transformedResource)(nil) // TODO(bep) resource + _ ReadSeekCloserResource = (*transformedResource)(nil) +) + +func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) { + return &transformedResource{Resource: r, transformation: t, cache: s.ResourceCache}, nil +} + +type ResourceTransformationCtx struct { + // The content to transform. + From io.Reader + + // The target of content transformation. + // The current implementation requires that r is written to w + // even if no transformation is performed. + To io.Writer + + // This is the relative target path to the resource. Unix styled slashes. + InPath string + + // The relative target path to the transformed resource. Unix styled slashes. + OutPath string + + // The input media type + InMediaType media.Type + + // The media type of the transformed resource. + OutMediaType media.Type + + // This is used to publis additional artifacts, e.g. source maps. + // We may improve this. + OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error) +} + +// AddOutPathIdentifier transforming InPath to OutPath adding an identifier, +// eg '.min' before any extension. +func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) { + ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier) +} + +func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string { + dir, file := path.Split(inPath) + base, ext := helpers.PathAndExt(file) + return path.Join(dir, (base + identifier + ext)) +} + +// ReplaceOutPathExtension transforming InPath to OutPath replacing the file +// extension, e.g. ".scss" +func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) { + dir, file := path.Split(ctx.InPath) + base, _ := helpers.PathAndExt(file) + ctx.OutPath = path.Join(dir, (base + newExt)) +} + +// PublishSourceMap writes the content to the target folder of the main resource +// with the ".map" extension added. +func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error { + target := ctx.OutPath + ".map" + f, err := ctx.OpenResourcePublisher(target) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write([]byte(content)) + return err +} + +// ResourceTransformationKey are provided by the different transformation implementations. +// It identifies the transformation (name) and its configuration (elements). +// We combine this in a chain with the rest of the transformations +// with the target filename and a content hash of the origin to use as cache key. +type ResourceTransformationKey struct { + name string + elements []interface{} +} + +// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation +// name and elements. We will create a 64 bit FNV hash from the elements, which when combined +// with the other key elements should be unique for all practical applications. +func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey { + return ResourceTransformationKey{name: name, elements: elements} +} + +// Do not change this without good reasons. +func (k ResourceTransformationKey) toString() string { + if len(k.elements) == 0 { + return k.name + } + + sb := bp.GetBuffer() + defer bp.PutBuffer(sb) + + sb.WriteString(k.name) + for _, element := range k.elements { + hash, err := hashstructure.Hash(element, nil) + if err != nil { + panic(err) + } + sb.WriteString("_") + sb.WriteString(strconv.FormatUint(hash, 10)) + } + + return sb.String() +} + +// ResourceTransformation is the interface that a resource transformation step +// needs to implement. +type ResourceTransformation interface { + Key() ResourceTransformationKey + Transform(ctx *ResourceTransformationCtx) error +} + +// We will persist this information to disk. +type transformedResourceMetadata struct { + Target string `json:"Target"` + MediaTypeV string `json:"MediaType"` +} + +// TODO(bep) resource how to handle fall back to cached assets when bootstrap submodule is missing +// in theme (shallow clone) etc. +type transformedResource struct { + cache *ResourceCache + + // This is the filename inside resources/_gen/assets + sourceFilename string + + linker permalinker + + // The transformation to apply. + transformation ResourceTransformation + + // We apply the tranformations lazily. + transformInit sync.Once + transformErr error + + // The transformed values + content string + contentInit sync.Once + transformedResourceMetadata + + // The source + Resource +} + +func (r *transformedResource) ReadSeekCloser() (ReadSeekCloser, error) { + rc, ok := r.Resource.(ReadSeekCloserResource) + if !ok { + return nil, fmt.Errorf("resource %T is not a ReadSeekerCloserResource", rc) + } + return rc.ReadSeekCloser() +} + +func (r *transformedResource) transferTransformedValues(another *transformedResource) { + if another.content != "" { + r.contentInit.Do(func() { + r.content = another.content + }) + } + r.transformedResourceMetadata = another.transformedResourceMetadata +} + +func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser { + f, meta, found := r.cache.getFromFile(key) + if !found { + return nil + } + r.transformedResourceMetadata = meta + r.sourceFilename = f.Name() + + return f +} + +func (r *transformedResource) Content() (interface{}, error) { + if err := r.initTransform(true); err != nil { + return nil, err + } + if err := r.initContent(); err != nil { + return "", err + } + return r.content, nil +} + +func (r *transformedResource) MediaType() media.Type { + if err := r.initTransform(false); err != nil { + return media.Type{} + } + m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV) + return m +} + +func (r *transformedResource) Permalink() string { + if err := r.initTransform(false); err != nil { + return "" + } + return r.linker.permalinkFor(r.Target) +} + +func (r *transformedResource) RelPermalink() string { + if err := r.initTransform(false); err != nil { + return "" + } + return r.linker.relPermalinkFor(r.Target) +} + +func (r *transformedResource) initContent() error { + var err error + r.contentInit.Do(func() { + var b []byte + b, err := afero.ReadFile(r.cache.rs.Resources.Fs, r.sourceFilename) + if err != nil { + return + } + r.content = string(b) + }) + return err +} + +func (r *transformedResource) transform(setContent bool) (err error) { + + // Used for source maps. + openPublishFileForWriting := func(relTargetPath string) (io.WriteCloser, error) { + return openFileForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathFor(relTargetPath)) + } + + // This can be the last resource in a chain. + // Rewind and create a processing chain. + var chain []Resource + current := r + for { + rr := current.Resource + chain = append(chain[:0], append([]Resource{rr}, chain[0:]...)...) + if tr, ok := rr.(*transformedResource); ok { + current = tr + } else { + break + } + } + + // Append the current transformer at the end + chain = append(chain, r) + + first := chain[0] + + contentrc, err := contentReadSeekerCloser(first) + if err != nil { + return err + } + defer contentrc.Close() + + var hash string + + if hasher, ok := first.(resourceHasher); ok { + if err = hasher.initHashFrom(contentrc); err != nil { + return + } + hash, err = hasher.getHash() + if err != nil { + return + } + } + + // Files with a suffix will be stored in cache (both on disk and in memory) + // partitioned by their suffix. There will be other files below /other. + // This partition is also how we determine what to delete on server reloads. + var key, base string + for _, element := range chain { + switch v := element.(type) { + case *transformedResource: + key = key + "_" + v.transformation.Key().toString() + case permalinker: + r.linker = v + p := v.relTargetPath() + partition := ResourceKeyPartition(p) + base = partition + "/" + p + default: + return fmt.Errorf("transformation not supported for type %T", element) + } + } + + key = path.Clean(base + "_" + helpers.MD5String(hash+key)) + cached, found := r.cache.get(key) + if found { + r.transferTransformedValues(cached.(*transformedResource)) + return + } + + // Acquire a write lock for the named transformation. + r.cache.nlocker.Lock(key) + // Check the cache again. + cached, found = r.cache.get(key) + if found { + r.transferTransformedValues(cached.(*transformedResource)) + r.cache.nlocker.Unlock(key) + return + } + defer r.cache.nlocker.Unlock(key) + defer r.cache.set(key, r) + + b1 := bp.GetBuffer() + b2 := bp.GetBuffer() + defer bp.PutBuffer(b1) + defer bp.PutBuffer(b2) + + tctx := &ResourceTransformationCtx{ + OpenResourcePublisher: openPublishFileForWriting, + } + + tctx.InMediaType = first.MediaType() + tctx.OutMediaType = first.MediaType() + tctx.From = contentrc + tctx.To = b1 + + if r.linker != nil { + tctx.InPath = r.linker.relTargetPath() + } + + counter := 0 + + var transformedContentr io.Reader + + for _, element := range chain { + tr, ok := element.(*transformedResource) + if !ok { + continue + } + counter++ + if counter != 1 { + tctx.InMediaType = tctx.OutMediaType + } + if counter%2 == 0 { + tctx.From = b1 + b2.Reset() + tctx.To = b2 + } else { + if counter != 1 { + // The first reader is the file. + tctx.From = b2 + } + b1.Reset() + tctx.To = b1 + } + + if err := tr.transformation.Transform(tctx); err != nil { + if err == errors.FeatureNotAvailableErr { + // This transformation is not available in this + // Hugo installation (scss not compiled in, PostCSS not available etc.) + // If a prepared bundle for this transformation chain is available, use that. + f := r.tryTransformedFileCache(key) + if f == nil { + return fmt.Errorf("failed to transform %q (%s): %s", tctx.InPath, tctx.InMediaType.Type(), err) + } + transformedContentr = f + defer f.Close() + } + } + + if tctx.OutPath != "" { + tctx.InPath = tctx.OutPath + tctx.OutPath = "" + } + } + + if transformedContentr == nil { + r.Target = tctx.InPath + r.MediaTypeV = tctx.OutMediaType.Type() + } + + publicw, err := openPublishFileForWriting(r.Target) + if err != nil { + r.transformErr = err + return + } + defer publicw.Close() + + publishwriters := []io.Writer{publicw} + + if transformedContentr == nil { + // Also write it to the cache + metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata) + if err != nil { + return err + } + r.sourceFilename = metaw.Name() + defer metaw.Close() + + publishwriters = append(publishwriters, metaw) + + if counter > 0 { + transformedContentr = tctx.To.(*bytes.Buffer) + } else { + transformedContentr = contentrc + } + } + + // Also write it to memory + var contentmemw *bytes.Buffer + + if setContent { + contentmemw = bp.GetBuffer() + defer bp.PutBuffer(contentmemw) + publishwriters = append(publishwriters, contentmemw) + } + + publishw := io.MultiWriter(publishwriters...) + _, r.transformErr = io.Copy(publishw, transformedContentr) + + if setContent { + r.contentInit.Do(func() { + r.content = contentmemw.String() + }) + } + + return nil + +} +func (r *transformedResource) initTransform(setContent bool) error { + r.transformInit.Do(func() { + if err := r.transform(setContent); err != nil { + r.transformErr = err + r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err) + } + }) + return r.transformErr +} + +// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource. +func contentReadSeekerCloser(r Resource) (ReadSeekCloser, error) { + switch rr := r.(type) { + case ReadSeekCloserResource: + rc, err := rr.ReadSeekCloser() + if err != nil { + return nil, err + } + return rc, nil + default: + return nil, fmt.Errorf("cannot tranform content of Resource of type %T", r) + + } +} diff --git a/resource/transform_test.go b/resource/transform_test.go new file mode 100644 index 00000000000..496e16bf672 --- /dev/null +++ b/resource/transform_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type testStruct struct { + Name string + V1 int64 + V2 int32 + V3 int + V4 uint64 +} + +func TestResourceTransformationKey(t *testing.T) { + // We really need this key to be portable across OSes. + key := NewResourceTransformationKey("testing", + testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)}) + assert := require.New(t) + assert.Equal(key.toString(), "testing_518996646957295636") +} diff --git a/source/filesystem_test.go b/source/filesystem_test.go index ee86c148742..2c1eeb171f5 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -75,12 +75,18 @@ func newTestConfig() *viper.Viper { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + v.Set("assetDir", "assets") return v } func newTestSourceSpec() *SourceSpec { v := newTestConfig() fs := hugofs.NewMem(v) - ps, _ := helpers.NewPathSpec(fs, v) + ps, err := helpers.NewPathSpec(fs, v) + if err != nil { + panic(err) + } return NewSourceSpec(ps, fs.Source) } diff --git a/tpl/os/init.go b/tpl/os/init.go index 012f43b1f62..3ef8702d6a2 100644 --- a/tpl/os/init.go +++ b/tpl/os/init.go @@ -37,14 +37,14 @@ func init() { ns.AddMethodMapping(ctx.ReadDir, []string{"readDir"}, [][2]string{ - {`{{ range (readDir ".") }}{{ .Name }}{{ end }}`, "README.txt"}, + {`{{ range (readDir "files") }}{{ .Name }}{{ end }}`, "README.txt"}, }, ) ns.AddMethodMapping(ctx.ReadFile, []string{"readFile"}, [][2]string{ - {`{{ readFile "README.txt" }}`, `Hugo Rocks!`}, + {`{{ readFile "files/README.txt" }}`, `Hugo Rocks!`}, }, ) diff --git a/tpl/os/os.go b/tpl/os/os.go index f7f9537ffed..79d035d7ea7 100644 --- a/tpl/os/os.go +++ b/tpl/os/os.go @@ -34,7 +34,7 @@ func New(deps *deps.Deps) *Namespace { if deps.Fs != nil { rfs = deps.Fs.WorkingDir if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil { - rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.ContentFs, deps.Fs.WorkingDir)) + rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.Content.Fs, deps.Fs.WorkingDir)) } } diff --git a/tpl/resources/init.go b/tpl/resources/init.go new file mode 100644 index 00000000000..7f8ee96d45b --- /dev/null +++ b/tpl/resources/init.go @@ -0,0 +1,68 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/tpl/internal" +) + +const name = "resources" + +func init() { + f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { + ctx, err := New(d) + if err != nil { + // TODO(bep) no panic. + panic(err) + } + + ns := &internal.TemplateFuncsNamespace{ + Name: name, + Context: func(args ...interface{}) interface{} { return ctx }, + } + + ns.AddMethodMapping(ctx.Open, + nil, + [][2]string{}, + ) + + // Add aliases for the most common transformations. + + ns.AddMethodMapping(ctx.Fingerprint, + []string{"fingerprint"}, + [][2]string{}, + ) + + ns.AddMethodMapping(ctx.Minify, + []string{"minify"}, + [][2]string{}, + ) + + ns.AddMethodMapping(ctx.ToCSS, + []string{"toCSS"}, + [][2]string{}, + ) + + ns.AddMethodMapping(ctx.PostCSS, + []string{"postCSS"}, + [][2]string{}, + ) + + return ns + + } + + internal.AddTemplateFuncsNamespace(f) +} diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go new file mode 100644 index 00000000000..d920db42cc9 --- /dev/null +++ b/tpl/resources/resources.go @@ -0,0 +1,204 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resource" + "github.com/gohugoio/hugo/resource/bundler" + "github.com/gohugoio/hugo/resource/create" + "github.com/gohugoio/hugo/resource/integrity" + "github.com/gohugoio/hugo/resource/minifiers" + "github.com/gohugoio/hugo/resource/postcss" + "github.com/gohugoio/hugo/resource/tocss/scss" + "github.com/spf13/cast" +) + +// New returns a new instance of the resources-namespaced template functions. +func New(deps *deps.Deps) (*Namespace, error) { + // TODO(bep) resource document + // There are currently lots of flexibility in where we look for resources + // but the SCSS import paths are currently restricted to /assets + scssClient, err := scss.New(deps.BaseFs.Assets, deps.ResourceSpec) + if err != nil { + return nil, err + } + return &Namespace{ + deps: deps, + scssClient: scssClient, + createClient: create.New(deps.ResourceSpec), + bundlerClient: bundler.New(deps.ResourceSpec), + integrityClient: integrity.New(deps.ResourceSpec), + minifyClient: minifiers.New(deps.ResourceSpec), + postcssClient: postcss.New(deps.ResourceSpec), + }, nil +} + +// Namespace provides template functions for the "resources" namespace. +type Namespace struct { + deps *deps.Deps + + createClient *create.Client + bundlerClient *bundler.Client + scssClient *scss.Client + integrityClient *integrity.Client + minifyClient *minifiers.Client + postcssClient *postcss.Client +} + +// Open locates the filename given in Hugo's filesystems: static, assets and content (in that order) +// and creates a Resource object that can be used for further transformations. +func (ns *Namespace) Open(filename interface{}) (resource.Resource, error) { + filenamestr, err := cast.ToStringE(filename) + if err != nil { + return nil, err + } + + filenamestr = filepath.Clean(filenamestr) + + // Locate the correct filesystem: static, assets and content (in that order) + _, fs, err := ns.deps.SourceFilesystems.StatResource(ns.deps.Lang(), filenamestr) + if err != nil { + return nil, err + } + return ns.createClient.Open(fs, filenamestr) + +} + +// ConcatTo concatenates a slice of Resource objects. These resources must +// (currently) be of the same Media Type. +func (ns *Namespace) ConcatTo(targetPathIn interface{}, r []interface{}) (resource.Resource, error) { + targetPath, err := cast.ToStringE(targetPathIn) + if err != nil { + return nil, err + } + rr := make([]resource.Resource, len(r)) + for i := 0; i < len(r); i++ { + rv, ok := r[i].(resource.Resource) + if !ok { + return nil, fmt.Errorf("cannot concat type %T", rv) + } + rr[i] = rv + } + return ns.bundlerClient.ConcatTo(targetPath, rr) +} + +// FromString creates a Resource from a string published to the relative target path. +// TODO(bep) resource we probably want the targetpath to be optional in these. +func (ns *Namespace) FromString(targetPathIn, contentIn interface{}) (resource.Resource, error) { + targetPath, err := cast.ToStringE(targetPathIn) + if err != nil { + return nil, err + } + content, err := cast.ToStringE(contentIn) + if err != nil { + return nil, err + } + + return ns.createClient.FromString(targetPath, content) +} + +// FromTemplate creates a Resource from a Go template, parsed and executed with +// the given data, and published to the relative target path. +// TODO(bep) resource we probably want the targetpath to be optional in these. +func (ns *Namespace) FromTemplate(targetPathIn, templIn, data interface{}) (resource.Resource, error) { + targetPath, err := cast.ToStringE(targetPathIn) + if err != nil { + return nil, err + } + templ, err := cast.ToStringE(templIn) + if err != nil { + return nil, err + } + + return ns.createClient.FromTemplate(targetPath, templ, data) +} + +// Fingerprint transforms the given Resource with a MD5 hash of the content in +// the RelPermalink and Permalink. +func (ns *Namespace) Fingerprint(r resource.Resource) (resource.Resource, error) { + return ns.integrityClient.Fingerprint(r) +} + +// Minify minifies the given Resource using the MediaType to pick the correct +// minifier. +func (ns *Namespace) Minify(r resource.Resource) (resource.Resource, error) { + return ns.minifyClient.Minify(r) +} + +// ToCSS converts the given Resource to CSS. You can optional provide an Options +// object as first argument. +func (ns *Namespace) ToCSS(args ...interface{}) (resource.Resource, error) { + r, m, err := ns.resolveArgs(args) + if err != nil { + return nil, err + } + var options scss.Options + if m != nil { + options, err = scss.DecodeOptions(m) + if err != nil { + return nil, err + } + } + + return ns.scssClient.ToCSS(r, options) +} + +// PostCSS processes the given Resource with PostCSS +func (ns *Namespace) PostCSS(args ...interface{}) (resource.Resource, error) { + r, m, err := ns.resolveArgs(args) + if err != nil { + return nil, err + } + var options postcss.Options + if m != nil { + options, err = postcss.DecodeOptions(m) + if err != nil { + return nil, err + } + } + + return ns.postcssClient.Process(r, options) +} + +// This roundabout way of doing it is needed to get both pipeline behaviour and options as arguments. +func (ns *Namespace) resolveArgs(args []interface{}) (resource.Resource, map[string]interface{}, error) { + if len(args) == 0 { + return nil, nil, errors.New("no Resource provided in transformation") + } + + if len(args) == 1 { + r, ok := args[0].(resource.Resource) + if !ok { + return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) + } + return r, nil, nil + } + + r, ok := args[1].(resource.Resource) + if !ok { + return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) + } + + m, err := cast.ToStringMapE(args[0]) + if err != nil { + return nil, nil, fmt.Errorf("invalid options type: %s", err) + } + + return r, m, nil +} diff --git a/tpl/resources/resources_test.go b/tpl/resources/resources_test.go new file mode 100644 index 00000000000..5339493d82c --- /dev/null +++ b/tpl/resources/resources_test.go @@ -0,0 +1,27 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestResource(t *testing.T) { + t.Parallel() + assert := require.New(t) + assert.True(true) + +} diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index 6ce387acafa..2b944fa1de4 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -35,6 +35,7 @@ import ( _ "github.com/gohugoio/hugo/tpl/os" _ "github.com/gohugoio/hugo/tpl/partials" _ "github.com/gohugoio/hugo/tpl/path" + _ "github.com/gohugoio/hugo/tpl/resources" _ "github.com/gohugoio/hugo/tpl/safe" _ "github.com/gohugoio/hugo/tpl/strings" _ "github.com/gohugoio/hugo/tpl/time" diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index a1745282dd2..9911b696ab1 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -51,6 +51,9 @@ func newTestConfig() config.Provider { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") return v } @@ -76,12 +79,13 @@ func TestTemplateFuncsExamples(t *testing.T) { v.Set("workingDir", workingDir) v.Set("multilingual", true) v.Set("contentDir", "content") + v.Set("assetDir", "assets") v.Set("baseURL", "http://mysite.com/hugo/") v.Set("CurrentContentLanguage", langs.NewLanguage("en", v)) fs := hugofs.NewMem(v) - afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(workingDir, "files", "README.txt"), []byte("Hugo Rocks!"), 0755) depsCfg := newDepsConfig(v) depsCfg.Fs = fs