tpl/transform: Add transform.Unmarshal func

Fixes #5428

cache/namedmemcache/named_cache.go

@@ -0,0 +1,84 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package namedmemcache provides a memory cache with a named lock. This is suitable
+// for situations where creating the cached resource can be time consuming or otherwise
+// resource hungry, or in situations where a "once only per key" is a requirement.
+package namedmemcache
+
+import (
+	"sync"
+
+	"github.com/BurntSushi/locker"
+)
+
+// Cache holds the cached values.
+type Cache struct {
+	nlocker *locker.Locker
+	cache   map[string]cacheEntry
+	mu      sync.RWMutex
+}
+
+type cacheEntry struct {
+	value interface{}
+	err   error
+}
+
+// New creates a new cache.
+func New() *Cache {
+	return &Cache{
+		nlocker: locker.NewLocker(),
+		cache:   make(map[string]cacheEntry),
+	}
+}
+
+// Clear clears the cache state.
+func (c *Cache) Clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.cache = make(map[string]cacheEntry)
+	c.nlocker = locker.NewLocker()
+
+}
+
+// GetOrCreate tries to get the value with the given cache key, if not found
+// create will be called and cached.
+// This method is thread safe. It also guarantees that the create func for a given
+// key is invoced only once for this cache.
+func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) {
+	c.mu.RLock()
+	entry, found := c.cache[key]
+	c.mu.RUnlock()
+
+	if found {
+		return entry.value, entry.err
+	}
+
+	c.nlocker.Lock(key)
+	defer c.nlocker.Unlock(key)
+
+	// Double check
+	if entry, found := c.cache[key]; found {
+		return entry.value, entry.err
+	}
+
+	// Create it.
+	value, err := create()
+
+	c.mu.Lock()
+	c.cache[key] = cacheEntry{value: value, err: err}
+	c.mu.Unlock()
+
+	return value, err
+}

cache/namedmemcache/named_cache_test.go

@@ -0,0 +1,80 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package namedmemcache
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNamedCache(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	cache := New()
+
+	counter := 0
+	create := func() (interface{}, error) {
+		counter++
+		return counter, nil
+	}
+
+	for i := 0; i < 5; i++ {
+		v1, err := cache.GetOrCreate("a1", create)
+		assert.NoError(err)
+		assert.Equal(1, v1)
+		v2, err := cache.GetOrCreate("a2", create)
+		assert.NoError(err)
+		assert.Equal(2, v2)
+	}
+
+	cache.Clear()
+
+	v3, err := cache.GetOrCreate("a2", create)
+	assert.NoError(err)
+	assert.Equal(3, v3)
+}
+
+func TestNamedCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	var wg sync.WaitGroup
+
+	cache := New()
+
+	create := func(i int) func() (interface{}, error) {
+		return func() (interface{}, error) {
+			return i, nil
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 100; j++ {
+				id := fmt.Sprintf("id%d", j)
+				v, err := cache.GetOrCreate(id, create(j))
+				assert.NoError(err)
+				assert.Equal(j, v)
+			}
+		}()
+	}
+	wg.Wait()
+}

deps/deps.go

@@ -123,6 +123,9 @@ type Listeners struct {
 
 // Add adds a function to a Listeners instance.
 func (b *Listeners) Add(f func()) {
+	if b == nil {
+		return
+	}
 	b.Lock()
 	defer b.Unlock()
 	b.listeners = append(b.listeners, f)
@@ -192,6 +195,14 @@ func New(cfg DepsCfg) (*Deps, error) {
 		fs = hugofs.NewDefault(cfg.Language)
 	}
 
+	if cfg.MediaTypes == nil {
+		cfg.MediaTypes = media.DefaultTypes
+	}
+
+	if cfg.OutputFormats == nil {
+		cfg.OutputFormats = output.DefaultFormats
+	}
+
 	ps, err := helpers.NewPathSpec(fs, cfg.Language)
 
 	if err != nil {

helpers/general.go

@@ -394,11 +394,10 @@ func MD5FromFileFast(r io.ReadSeeker) (string, error) {
 	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
-// MD5FromFile creates a MD5 hash from the given file.
-// It will not close the file.
-func MD5FromFile(f afero.File) (string, error) {
+// MD5FromReader creates a MD5 hash from the given reader.
+func MD5FromReader(r io.Reader) (string, error) {
 	h := md5.New()
-	if _, err := io.Copy(h, f); err != nil {
+	if _, err := io.Copy(h, r); err != nil {
 		return "", nil
 	}
 	return hex.EncodeToString(h.Sum(nil)), nil

helpers/general_test.go

@@ -272,7 +272,7 @@ func TestFastMD5FromFile(t *testing.T) {
 	req.NoError(err)
 	req.NotEqual(m3, m4)
 
-	m5, err := MD5FromFile(bf2)
+	m5, err := MD5FromReader(bf2)
 	req.NoError(err)
 	req.NotEqual(m4, m5)
 }
@@ -293,7 +293,7 @@ func BenchmarkMD5FromFileFast(b *testing.B) {
 				}
 				b.StartTimer()
 				if full {
-					if _, err := MD5FromFile(f); err != nil {
+					if _, err := MD5FromReader(f); err != nil {
 						b.Fatal(err)
 					}
 				} else {

hugolib/resource_chain_test.go

@@ -339,6 +339,16 @@ Publish 2: {{ $cssPublish2.Permalink }}
 			assert.False(b.CheckExists("public/inline.min.css"), "Inline content should not be copied to /public")
 		}},
 
+		{"unmarshal", func() bool { return true }, func(b *sitesBuilder) {
+			b.WithTemplates("home.html", `
+{{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }}
+Slogan: {{ $toml.slogan }}
+
+`)
+		}, func(b *sitesBuilder) {
+			b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`)
+		}},
+
 		{"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) {
 		}},
 	}

media/mediaType.go

@@ -135,6 +135,8 @@ var (
 	XMLType        = Type{MainType: "application", SubType: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter}
 	SVGType        = Type{MainType: "image", SubType: "svg", mimeSuffix: "xml", Suffixes: []string{"svg"}, Delimiter: defaultDelimiter}
 	TextType       = Type{MainType: "text", SubType: "plain", Suffixes: []string{"txt"}, Delimiter: defaultDelimiter}
+	TOMLType       = Type{MainType: "application", SubType: "toml", Suffixes: []string{"toml"}, Delimiter: defaultDelimiter}
+	YAMLType       = Type{MainType: "application", SubType: "yaml", Suffixes: []string{"yaml", "yml"}, Delimiter: defaultDelimiter}
 
 	OctetType = Type{MainType: "application", SubType: "octet-stream"}
 )
@@ -154,6 +156,8 @@ var DefaultTypes = Types{
 	SVGType,
 	TextType,
 	OctetType,
+	YAMLType,
+	TOMLType,
 }
 
 func init() {

media/mediaType_test.go

@@ -39,6 +39,8 @@ func TestDefaultTypes(t *testing.T) {
 		{SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"},
 		{TextType, "text", "plain", "txt", "text/plain", "text/plain"},
 		{XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
+		{TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
+		{YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
 	} {
 		require.Equal(t, test.expectedMainType, test.tp.MainType)
 		require.Equal(t, test.expectedSubType, test.tp.SubType)
@@ -50,6 +52,8 @@ func TestDefaultTypes(t *testing.T) {
 
 	}
 
+	require.Equal(t, 15, len(DefaultTypes))
+
 }
 
 func TestGetByType(t *testing.T) {

parser/metadecoders/format.go

@@ -17,6 +17,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/gohugoio/hugo/media"
+
 	"github.com/gohugoio/hugo/parser/pageparser"
 )
 
@@ -55,6 +57,18 @@ func FormatFromString(formatStr string) Format {
 
 }
 
+// FormatFromMediaType gets the Format given a MIME type, empty string
+// if unknown.
+func FormatFromMediaType(m media.Type) Format {
+	for _, suffix := range m.Suffixes {
+		if f := FormatFromString(suffix); f != "" {
+			return f
+		}
+	}
+
+	return ""
+}
+
 // FormatFromFrontMatterType will return empty if not supported.
 func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
 	switch typ {
@@ -70,3 +84,39 @@ func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
 		return ""
 	}
 }
+
+// FormatFromContentString tries to detect the format (JSON, YAML or TOML)
+// in the given string.
+// It return an empty string if no format could be detected.
+func FormatFromContentString(data string) Format {
+	jsonIdx := strings.Index(data, "{")
+	yamlIdx := strings.Index(data, ":")
+	tomlIdx := strings.Index(data, "=")
+
+	if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) {
+		return JSON
+	}
+
+	if isLowerIndexThan(yamlIdx, tomlIdx) {
+		return YAML
+	}
+
+	if tomlIdx != -1 {
+		return TOML
+	}
+
+	return ""
+}
+
+func isLowerIndexThan(first int, others ...int) bool {
+	if first == -1 {
+		return false
+	}
+	for _, other := range others {
+		if other != -1 && other < first {
+			return false
+		}
+	}
+
+	return true
+}