From 88f875661aacc2c82084e7b929cdc30def5eee03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 21 Feb 2018 11:09:08 +0100 Subject: [PATCH] hugolib: Extract date and slug from filename This commit adds a new config option which, when enabled and no date is set in front matter, will make Hugo try to parse the date from the content filename. Also, the filenames in these cases will make for very poor permalinks, so we will also use the remaining part as the page `slug` if that value is not set in front matter. This should make it easier to move content from Jekyll to Hugo. To enable, put this in your `config.toml`: ```toml [frontmatter] defaultDate = ["filename"] ``` Fixes #285 Closes #3310 Closes #3762 Closes #4340 --- hugolib/page.go | 35 +++++++-- hugolib/page_frontmatter.go | 117 +++++++++++++++++++++++++++++++ hugolib/page_frontmatter_test.go | 39 +++++++++++ hugolib/page_test.go | 58 ++++++++++++++- hugolib/site.go | 10 +++ 5 files changed, 252 insertions(+), 7 deletions(-) create mode 100644 hugolib/page_frontmatter.go create mode 100644 hugolib/page_frontmatter_test.go diff --git a/hugolib/page.go b/hugolib/page.go index fd6278bb443..900c05d1530 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -1115,6 +1115,9 @@ func (p *Page) update(frontmatter map[string]interface{}) error { // Needed for case insensitive fetching of params values helpers.ToLowerMap(frontmatter) + // Handle the date separately + p.s.frontmatterConfig.handleDate(frontmatter, p) + var modified time.Time var err error @@ -1151,11 +1154,7 @@ func (p *Page) update(frontmatter map[string]interface{}) error { p.Keywords = cast.ToStringSlice(v) p.params[loki] = p.Keywords case "date": - p.Date, err = cast.ToTimeE(v) - if err != nil { - p.s.Log.ERROR.Printf("Failed to parse date '%v' in page %s", v, p.File.Path()) - } - p.params[loki] = p.Date + // Handled separately. case "headless": // For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output). // We may expand on this in the future, but that gets more complex pretty fast. @@ -1373,6 +1372,30 @@ func (p *Page) update(frontmatter map[string]interface{}) error { return nil } +// A Zero date is a signal that the name can not be parsed. +// This follows the format as outlined in Jekyll, https://jekyllrb.com/docs/posts/: +// "Where YEAR is a four-digit number, MONTH and DAY are both two-digit numbers" +func dateAndSlugFromBaseFilename(name string) (time.Time, string) { + withoutExt, _ := helpers.FileAndExt(name) + + if len(withoutExt) < 10 { + // This can not be a date. + return time.Time{}, "" + } + + // Note: Hugo currently have no custom timezone support. + // We will have to revisit this when that is in place. + d, err := time.Parse("2006-01-02", withoutExt[:10]) + if err != nil { + return time.Time{}, "" + } + + // Be a little lenient with the format here. + slug := strings.Trim(withoutExt[10:], " -_") + + return d, slug +} + func (p *Page) GetParam(key string) interface{} { return p.getParam(key, false) } diff --git a/hugolib/page_frontmatter.go b/hugolib/page_frontmatter.go new file mode 100644 index 00000000000..b86c5865494 --- /dev/null +++ b/hugolib/page_frontmatter.go @@ -0,0 +1,117 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "strings" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/cast" + jww "github.com/spf13/jwalterweatherman" +) + +// TODO(bep) should probably make the date handling chain complete to give people the flexibility they want. + +type frontmatterConfig struct { + // Ordered chain. + dateHandlers []frontmatterFieldHandler + + logger *jww.Notepad +} + +func (f frontmatterConfig) handleField(handlers []frontmatterFieldHandler, frontmatter map[string]interface{}, p *Page) { + for _, h := range handlers { + handled, err := h(frontmatter, p) + if err != nil { + f.logger.ERROR.Println(err) + } + if handled { + break + } + } +} + +func (f frontmatterConfig) handleDate(frontmatter map[string]interface{}, p *Page) { + f.handleField(f.dateHandlers, frontmatter, p) +} + +type frontmatterFieldHandler func(frontmatter map[string]interface{}, p *Page) (bool, error) + +func newFrontmatterConfig(logger *jww.Notepad, cfg config.Provider) (frontmatterConfig, error) { + + if logger == nil { + logger = jww.NewNotepad(jww.LevelWarn, jww.LevelWarn, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + } + + f := frontmatterConfig{logger: logger} + + handlers := &frontmatterFieldHandlers{logger: logger} + + f.dateHandlers = []frontmatterFieldHandler{handlers.defaultDateHandler} + + if cfg.IsSet("frontmatter") { + fm := cfg.GetStringMap("frontmatter") + if fm != nil { + dateFallbacks, found := fm["defaultdate"] + if found { + slice, err := cast.ToStringSliceE(dateFallbacks) + if err != nil { + return f, fmt.Errorf("invalid value for dataCallbacks, expeced a string slice, got %T", dateFallbacks) + } + + for _, v := range slice { + if strings.EqualFold(v, "filename") { + f.dateHandlers = append(f.dateHandlers, handlers.fileanameFallbackDateHandler) + // No more for now. + break + } + } + } + } + } + + return f, nil +} + +type frontmatterFieldHandlers struct { + logger *jww.Notepad +} + +// TODO(bep) modtime + +func (f *frontmatterFieldHandlers) defaultDateHandler(frontmatter map[string]interface{}, p *Page) (bool, error) { + loki := "date" + v, found := frontmatter[loki] + if !found { + return false, nil + } + + var err error + p.Date, err = cast.ToTimeE(v) + if err != nil { + return false, fmt.Errorf("Failed to parse date %q in page %s", v, p.File.Path()) + } + + p.params[loki] = p.Date + + return true, nil +} + +func (f *frontmatterFieldHandlers) fileanameFallbackDateHandler(frontmatter map[string]interface{}, p *Page) (bool, error) { + return true, nil +} diff --git a/hugolib/page_frontmatter_test.go b/hugolib/page_frontmatter_test.go new file mode 100644 index 00000000000..1624de56a0e --- /dev/null +++ b/hugolib/page_frontmatter_test.go @@ -0,0 +1,39 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestNewFrontmatterConfig(t *testing.T) { + t.Parallel() + + v := viper.New() + + v.Set("frontmatter", map[string]interface{}{ + "defaultDate": []string{"filename"}, + }) + + assert := require.New(t) + + fc, err := newFrontmatterConfig(newWarningLogger(), v) + + assert.NoError(err) + assert.Equal(2, len(fc.dateHandlers)) + +} diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 814556c6c59..b5f97caac80 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -985,6 +985,25 @@ Page With empty front matter` zero_FM = "Page With empty front matter" ) +/*func TestPageWithFilenameDateAsFallback(t *testing.T) { + t.Parallel() + cfg, fs := newTestCfg() + + var tests = []struct { + }{} + + writeSource(t, fs, filepath.Join("content", "simple.md"), simplePageRFC3339Date) + + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + + require.Len(t, s.RegularPages, 1) + + p := s.RegularPages[0] + d, _ := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z") + + checkPageDate(t, p, d) +} +*/ func TestMetadataDates(t *testing.T) { t.Parallel() var tests = []struct { @@ -1873,6 +1892,43 @@ tags: } } +func TestDateAndSlugFromBaseFilename(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + tests := []struct { + name string + date string + slug string + }{ + {"page.md", "0001-01-01", ""}, + {"2012-09-12-page.md", "2012-09-12", "page"}, + {"2018-02-28-page.md", "2018-02-28", "page"}, + {"2018-02-28_page.md", "2018-02-28", "page"}, + {"2018-02-28 page.md", "2018-02-28", "page"}, + {"2018-02-28page.md", "2018-02-28", "page"}, + {"2018-02-28-.md", "2018-02-28", ""}, + {"2018-02-28-.md", "2018-02-28", ""}, + {"2018-02-28.md", "2018-02-28", ""}, + {"2018-02-28-page", "2018-02-28", "page"}, + {"2012-9-12-page.md", "0001-01-01", ""}, + {"asdfasdf.md", "0001-01-01", ""}, + } + + for i, test := range tests { + expectedDate, err := time.Parse("2006-01-02", test.date) + assert.NoError(err) + + errMsg := fmt.Sprintf("Test %d", i) + gotDate, gotSlug := dateAndSlugFromBaseFilename(test.name) + + assert.Equal(expectedDate, gotDate, errMsg) + assert.Equal(test.slug, gotSlug, errMsg) + + } +} + func BenchmarkParsePage(b *testing.B) { s := newTestSite(b) f, _ := os.Open("testdata/redis.cn.md") diff --git a/hugolib/site.go b/hugolib/site.go index 95cd0a23edf..6fdde99336b 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -121,6 +121,9 @@ type Site struct { outputFormatsConfig output.Formats mediaTypesConfig media.Types + // How to handle page front matter. + frontmatterConfig frontmatterConfig + // We render each site for all the relevant output formats in serial with // this rendering context pointing to the current one. rc *siteRenderingContext @@ -177,6 +180,7 @@ func (s *Site) reset() *Site { relatedDocsHandler: newSearchIndexHandler(s.relatedDocsHandler.cfg), outputFormats: s.outputFormats, outputFormatsConfig: s.outputFormatsConfig, + frontmatterConfig: s.frontmatterConfig, mediaTypesConfig: s.mediaTypesConfig, resourceSpec: s.resourceSpec, Language: s.Language, @@ -248,6 +252,11 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { titleFunc := helpers.GetTitleFunc(cfg.Language.GetString("titleCaseStyle")) + fmConfig, err := newFrontmatterConfig(cfg.Logger, cfg.Cfg) + if err != nil { + return nil, err + } + s := &Site{ PageCollections: c, layoutHandler: output.NewLayoutHandler(cfg.Cfg.GetString("themesDir") != ""), @@ -258,6 +267,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { outputFormats: outputFormats, outputFormatsConfig: siteOutputFormatsConfig, mediaTypesConfig: siteMediaTypesConfig, + frontmatterConfig: fmConfig, } s.Info = newSiteInfo(siteBuilderCfg{s: s, pageCollections: c, language: s.Language})