Permalink
Browse files

tpl/transform: Add transform.Unmarshal func

Fixes #5428
  • Loading branch information...
bep committed Dec 21, 2018
1 parent 43f9df0 commit 822dc627a1cfdf1f97882f27761675ac6ace7669
@@ -0,0 +1,84 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package namedmemcache provides a memory cache with a named lock. This is suitable
// for situations where creating the cached resource can be time consuming or otherwise
// resource hungry, or in situations where a "once only per key" is a requirement.
package namedmemcache

import (
"sync"

"github.com/BurntSushi/locker"
)

// Cache holds the cached values.
type Cache struct {
nlocker *locker.Locker
cache map[string]cacheEntry
mu sync.RWMutex
}

type cacheEntry struct {
value interface{}
err error
}

// New creates a new cache.
func New() *Cache {
return &Cache{
nlocker: locker.NewLocker(),
cache: make(map[string]cacheEntry),
}
}

// Clear clears the cache state.
func (c *Cache) Clear() {
c.mu.Lock()
defer c.mu.Unlock()

c.cache = make(map[string]cacheEntry)
c.nlocker = locker.NewLocker()

}

// GetOrCreate tries to get the value with the given cache key, if not found
// create will be called and cached.
// This method is thread safe. It also guarantees that the create func for a given
// key is invoced only once for this cache.
func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) {
c.mu.RLock()
entry, found := c.cache[key]
c.mu.RUnlock()

if found {
return entry.value, entry.err
}

c.nlocker.Lock(key)
defer c.nlocker.Unlock(key)

// Double check
if entry, found := c.cache[key]; found {
return entry.value, entry.err
}

// Create it.
value, err := create()

c.mu.Lock()
c.cache[key] = cacheEntry{value: value, err: err}
c.mu.Unlock()

return value, err
}
@@ -0,0 +1,80 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package namedmemcache

import (
"fmt"
"sync"
"testing"

"github.com/stretchr/testify/require"
)

func TestNamedCache(t *testing.T) {
t.Parallel()
assert := require.New(t)

cache := New()

counter := 0
create := func() (interface{}, error) {
counter++
return counter, nil
}

for i := 0; i < 5; i++ {
v1, err := cache.GetOrCreate("a1", create)
assert.NoError(err)
assert.Equal(1, v1)
v2, err := cache.GetOrCreate("a2", create)
assert.NoError(err)
assert.Equal(2, v2)
}

cache.Clear()

v3, err := cache.GetOrCreate("a2", create)
assert.NoError(err)
assert.Equal(3, v3)
}

func TestNamedCacheConcurrent(t *testing.T) {
t.Parallel()

assert := require.New(t)

var wg sync.WaitGroup

cache := New()

create := func(i int) func() (interface{}, error) {
return func() (interface{}, error) {
return i, nil
}
}

for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 100; j++ {
id := fmt.Sprintf("id%d", j)
v, err := cache.GetOrCreate(id, create(j))
assert.NoError(err)
assert.Equal(j, v)
}
}()
}
wg.Wait()
}
@@ -123,6 +123,9 @@ type Listeners struct {

// Add adds a function to a Listeners instance.
func (b *Listeners) Add(f func()) {
if b == nil {
return
}
b.Lock()
defer b.Unlock()
b.listeners = append(b.listeners, f)
@@ -192,6 +195,14 @@ func New(cfg DepsCfg) (*Deps, error) {
fs = hugofs.NewDefault(cfg.Language)
}

if cfg.MediaTypes == nil {
cfg.MediaTypes = media.DefaultTypes
}

if cfg.OutputFormats == nil {
cfg.OutputFormats = output.DefaultFormats
}

ps, err := helpers.NewPathSpec(fs, cfg.Language)

if err != nil {
@@ -394,11 +394,10 @@ func MD5FromFileFast(r io.ReadSeeker) (string, error) {
return hex.EncodeToString(h.Sum(nil)), nil
}

// MD5FromFile creates a MD5 hash from the given file.
// It will not close the file.
func MD5FromFile(f afero.File) (string, error) {
// MD5FromReader creates a MD5 hash from the given reader.
func MD5FromReader(r io.Reader) (string, error) {
h := md5.New()
if _, err := io.Copy(h, f); err != nil {
if _, err := io.Copy(h, r); err != nil {
return "", nil
}
return hex.EncodeToString(h.Sum(nil)), nil
@@ -272,7 +272,7 @@ func TestFastMD5FromFile(t *testing.T) {
req.NoError(err)
req.NotEqual(m3, m4)

m5, err := MD5FromFile(bf2)
m5, err := MD5FromReader(bf2)
req.NoError(err)
req.NotEqual(m4, m5)
}
@@ -293,7 +293,7 @@ func BenchmarkMD5FromFileFast(b *testing.B) {
}
b.StartTimer()
if full {
if _, err := MD5FromFile(f); err != nil {
if _, err := MD5FromReader(f); err != nil {
b.Fatal(err)
}
} else {
@@ -339,6 +339,16 @@ Publish 2: {{ $cssPublish2.Permalink }}
assert.False(b.CheckExists("public/inline.min.css"), "Inline content should not be copied to /public")
}},

{"unmarshal", func() bool { return true }, func(b *sitesBuilder) {
b.WithTemplates("home.html", `
{{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }}
Slogan: {{ $toml.slogan }}
`)
}, func(b *sitesBuilder) {
b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`)
}},

{"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) {
}},
}
@@ -135,6 +135,8 @@ var (
XMLType = Type{MainType: "application", SubType: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter}
SVGType = Type{MainType: "image", SubType: "svg", mimeSuffix: "xml", Suffixes: []string{"svg"}, Delimiter: defaultDelimiter}
TextType = Type{MainType: "text", SubType: "plain", Suffixes: []string{"txt"}, Delimiter: defaultDelimiter}
TOMLType = Type{MainType: "application", SubType: "toml", Suffixes: []string{"toml"}, Delimiter: defaultDelimiter}
YAMLType = Type{MainType: "application", SubType: "yaml", Suffixes: []string{"yaml", "yml"}, Delimiter: defaultDelimiter}

OctetType = Type{MainType: "application", SubType: "octet-stream"}
)
@@ -154,6 +156,8 @@ var DefaultTypes = Types{
SVGType,
TextType,
OctetType,
YAMLType,
TOMLType,
}

func init() {
@@ -39,6 +39,8 @@ func TestDefaultTypes(t *testing.T) {
{SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"},
{TextType, "text", "plain", "txt", "text/plain", "text/plain"},
{XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
{TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
{YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
} {
require.Equal(t, test.expectedMainType, test.tp.MainType)
require.Equal(t, test.expectedSubType, test.tp.SubType)
@@ -50,6 +52,8 @@ func TestDefaultTypes(t *testing.T) {

}

require.Equal(t, 15, len(DefaultTypes))

}

func TestGetByType(t *testing.T) {
@@ -17,6 +17,8 @@ import (
"path/filepath"
"strings"

"github.com/gohugoio/hugo/media"

"github.com/gohugoio/hugo/parser/pageparser"
)

@@ -55,6 +57,18 @@ func FormatFromString(formatStr string) Format {

}

// FormatFromMediaType gets the Format given a MIME type, empty string
// if unknown.
func FormatFromMediaType(m media.Type) Format {
for _, suffix := range m.Suffixes {
if f := FormatFromString(suffix); f != "" {
return f
}
}

return ""
}

// FormatFromFrontMatterType will return empty if not supported.
func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
switch typ {
@@ -70,3 +84,39 @@ func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
return ""
}
}

// FormatFromContentString tries to detect the format (JSON, YAML or TOML)
// in the given string.
// It return an empty string if no format could be detected.
func FormatFromContentString(data string) Format {
jsonIdx := strings.Index(data, "{")
yamlIdx := strings.Index(data, ":")
tomlIdx := strings.Index(data, "=")

if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) {
return JSON
}

if isLowerIndexThan(yamlIdx, tomlIdx) {
return YAML
}

if tomlIdx != -1 {
return TOML
}

return ""
}

func isLowerIndexThan(first int, others ...int) bool {
if first == -1 {
return false
}
for _, other := range others {
if other != -1 && other < first {
return false
}
}

return true
}
Oops, something went wrong.

0 comments on commit 822dc62

Please sign in to comment.