diff --git a/helpers/general.go b/helpers/general.go index da05548f49a..e6c0ec8e574 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -345,11 +345,11 @@ func InitLoggers() { // plenty of time to fix their templates. func Deprecated(object, item, alternative string, err bool) { if err { - DistinctErrorLog.Printf("%s's %s is deprecated and will be removed in Hugo %s. %s.", object, item, CurrentHugoVersion.Next().ReleaseVersion(), alternative) + DistinctErrorLog.Printf("%s's %s is deprecated and will be removed in Hugo %s. %s", object, item, CurrentHugoVersion.Next().ReleaseVersion(), alternative) } else { // Make sure the users see this while avoiding build breakage. This will not lead to an os.Exit(-1) - DistinctFeedbackLog.Printf("WARNING: %s's %s is deprecated and will be removed in a future release. %s.", object, item, alternative) + DistinctFeedbackLog.Printf("WARNING: %s's %s is deprecated and will be removed in a future release. %s", object, item, alternative) } } diff --git a/hugolib/config.go b/hugolib/config.go index fc9aeb5ae2d..b74a61fba86 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -249,9 +249,16 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("debug", false) v.SetDefault("disableFastRender", false) - // Remove in Hugo 0.37 + // Remove in Hugo 0.39 + if v.GetBool("useModTimeAsFallback") { - helpers.Deprecated("Site config", "useModTimeAsFallback", "Try --enableGitInfo or set lastMod in front matter", false) + + helpers.Deprecated("Site config", "useModTimeAsFallback", `Replace with this in your config.toml: + +[frontmatter] +date = [ "date",":fileModTime"] +lastMod = ["lastMod" ,":fileModTime", "date"] +`, false) } diff --git a/hugolib/page.go b/hugolib/page.go index fd6278bb443..2274aa84ae8 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import ( "github.com/bep/gitmap" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugolib/pagemeta" "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/output" @@ -140,9 +141,6 @@ type Page struct { Draft bool Status string - PublishDate time.Time - ExpiryDate time.Time - // PageMeta contains page stats such as word count etc. PageMeta @@ -223,11 +221,12 @@ type Page struct { Keywords []string Data map[string]interface{} - Date time.Time - Lastmod time.Time + pagemeta.PageDates Sitemap Sitemap - URLPath + pagemeta.URLPath + frontMatterURL string + permalink string relPermalink string @@ -1115,12 +1114,44 @@ func (p *Page) update(frontmatter map[string]interface{}) error { // Needed for case insensitive fetching of params values helpers.ToLowerMap(frontmatter) - var modified time.Time + var mtime time.Time + if p.Source.FileInfo() != nil { + mtime = p.Source.FileInfo().ModTime() + } + + descriptor := &pagemeta.FrontMatterDescriptor{ + Frontmatter: frontmatter, + Params: p.params, + Dates: &p.PageDates, + PageURLs: &p.URLPath, + BaseFilename: p.BaseFileName(), + ModTime: mtime} + + // Handle the date separately + // TODO(bep) we need to "do more" in this area so this can be split up and + // more easily tested without the Page, but the coupling is strong. + err := p.s.frontmatterHandler.HandleDates(descriptor) + if err != nil { + p.s.Log.ERROR.Printf("Failed to handle dates for page %q: %s", p.Path(), err) + } - var err error var draft, published, isCJKLanguage *bool for k, v := range frontmatter { loki := strings.ToLower(k) + + if loki == "published" { // Intentionally undocumented + vv, err := cast.ToBoolE(v) + if err == nil { + published = &vv + } + // published may also be a date + continue + } + + if p.s.frontmatterHandler.IsDateKey(loki) { + continue + } + switch loki { case "title": p.title = cast.ToString(v) @@ -1139,7 +1170,7 @@ func (p *Page) update(frontmatter map[string]interface{}) error { return fmt.Errorf("Only relative URLs are supported, %v provided", url) } p.URLPath.URL = cast.ToString(v) - p.URLPath.frontMatterURL = p.URLPath.URL + p.frontMatterURL = p.URLPath.URL p.params[loki] = p.URLPath.URL case "type": p.contentType = cast.ToString(v) @@ -1150,12 +1181,6 @@ func (p *Page) update(frontmatter map[string]interface{}) error { case "keywords": p.Keywords = cast.ToStringSlice(v) p.params[loki] = p.Keywords - case "date": - p.Date, err = cast.ToTimeE(v) - if err != nil { - p.s.Log.ERROR.Printf("Failed to parse date '%v' in page %s", v, p.File.Path()) - } - p.params[loki] = p.Date case "headless": // For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output). // We may expand on this in the future, but that gets more complex pretty fast. @@ -1163,19 +1188,6 @@ func (p *Page) update(frontmatter map[string]interface{}) error { p.headless = cast.ToBool(v) } p.params[loki] = p.headless - case "lastmod": - p.Lastmod, err = cast.ToTimeE(v) - if err != nil { - p.s.Log.ERROR.Printf("Failed to parse lastmod '%v' in page %s", v, p.File.Path()) - } - case "modified": - vv, err := cast.ToTimeE(v) - if err == nil { - p.params[loki] = vv - modified = vv - } else { - p.params[loki] = cast.ToString(v) - } case "outputs": o := cast.ToStringSlice(v) if len(o) > 0 { @@ -1190,34 +1202,9 @@ func (p *Page) update(frontmatter map[string]interface{}) error { } } - case "publishdate", "pubdate": - p.PublishDate, err = cast.ToTimeE(v) - if err != nil { - p.s.Log.ERROR.Printf("Failed to parse publishdate '%v' in page %s", v, p.File.Path()) - } - p.params[loki] = p.PublishDate - case "expirydate", "unpublishdate": - p.ExpiryDate, err = cast.ToTimeE(v) - if err != nil { - p.s.Log.ERROR.Printf("Failed to parse expirydate '%v' in page %s", v, p.File.Path()) - } case "draft": draft = new(bool) *draft = cast.ToBool(v) - case "published": // Intentionally undocumented - vv, err := cast.ToBoolE(v) - if err == nil { - published = &vv - } else { - // Some sites use this as the publishdate - vv, err := cast.ToTimeE(v) - if err == nil { - p.PublishDate = vv - p.params[loki] = p.PublishDate - } else { - p.params[loki] = cast.ToString(v) - } - } case "layout": p.Layout = cast.ToString(v) p.params[loki] = p.Layout @@ -1333,32 +1320,6 @@ func (p *Page) update(frontmatter map[string]interface{}) error { } p.params["draft"] = p.Draft - if p.Date.IsZero() { - p.Date = p.PublishDate - } - - if p.PublishDate.IsZero() { - p.PublishDate = p.Date - } - - if p.Date.IsZero() && p.s.Cfg.GetBool("useModTimeAsFallback") { - p.Date = p.Source.FileInfo().ModTime() - } - - if p.Lastmod.IsZero() { - if !modified.IsZero() { - p.Lastmod = modified - } else { - p.Lastmod = p.Date - } - - } - - p.params["date"] = p.Date - p.params["lastmod"] = p.Lastmod - p.params["publishdate"] = p.PublishDate - p.params["expirydate"] = p.ExpiryDate - if isCJKLanguage != nil { p.isCJKLanguage = *isCJKLanguage } else if p.s.Cfg.GetBool("hasCJKLanguage") { @@ -1865,14 +1826,6 @@ func (p *Page) String() string { return fmt.Sprintf("Page(%q)", p.title) } -type URLPath struct { - URL string - frontMatterURL string - Permalink string - Slug string - Section string -} - // Scratch returns the writable context associated with this Page. func (p *Page) Scratch() *Scratch { if p.scratch == nil { diff --git a/hugolib/page_paths.go b/hugolib/page_paths.go index ce8a700b1c5..4d64f4c1488 100644 --- a/hugolib/page_paths.go +++ b/hugolib/page_paths.go @@ -88,7 +88,7 @@ func (p *Page) initTargetPathDescriptor() error { Sections: p.sections, UglyURLs: p.s.Info.uglyURLs(p), Dir: filepath.ToSlash(p.Source.Dir()), - URL: p.URLPath.frontMatterURL, + URL: p.frontMatterURL, IsMultihost: p.s.owner.IsMultihost(), } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 814556c6c59..905793ca601 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,8 +27,6 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/source" "github.com/spf13/cast" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -728,6 +726,7 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { } // Issue #3854 +// Also see https://github.com/gohugoio/hugo/issues/3977 func TestPageWithDateFields(t *testing.T) { assert := require.New(t) pageWithDate := `--- @@ -737,8 +736,8 @@ weight: %d --- Simple Page With Some Date` - hasBothDates := func(p *Page) bool { - return p.Date.Year() == 2017 && p.PublishDate.Year() == 2017 + hasDate := func(p *Page) bool { + return p.Date.Year() == 2017 } datePage := func(field string, weight int) string { @@ -749,7 +748,7 @@ Simple Page With Some Date` assertFunc := func(t *testing.T, ext string, pages Pages) { assert.True(len(pages) > 0) for _, p := range pages { - assert.True(hasBothDates(p)) + assert.True(hasDate(p)) } } @@ -905,186 +904,68 @@ func TestPageWithDate(t *testing.T) { checkPageDate(t, p, d) } -const ( - s = "fs mod timestamp" // signifies filesystem's modification timestamp - P = "1969-01-10T09:17:42Z" - D = "2013-10-15T06:16:13Z" - L = "2017-09-03T22:22:22Z" - M = "2018-01-24T12:21:39Z" - E = "2025-12-31T23:59:59Z" - o = "0001-01-01T00:00:00Z" // zero value of type Time, default for some date fields - x = "" // nil date value, default for some date fields - - p_D____ = `--- -title: Simple -date: '2013-10-15T06:16:13' ---- -Page With Date only` - - p__P___ = `--- -title: Simple -publishdate: '1969-01-10T09:17:42' ---- -Page With PublishDate only` - - p_DP___ = `--- -title: Simple -date: '2013-10-15T06:16:13' -publishdate: '1969-01-10T09:17:42' ---- -Page With Date and PublishDate` - - p__PL__ = `--- -title: Simple -publishdate: '1969-01-10T09:17:42' -lastmod: '2017-09-03T22:22:22' ---- -Page With Date and PublishDate` - - p_DPL__ = `--- -title: Simple -date: '2013-10-15T06:16:13' -publishdate: '1969-01-10T09:17:42' -lastmod: '2017-09-03T22:22:22' ---- -Page With Date, PublishDate and LastMod` +func TestPageWithFrontMatterConfig(t *testing.T) { + t.Parallel() - p_DPL_E = `--- -title: Simple -date: '2013-10-15T06:16:13' -publishdate: '1969-01-10T09:17:42' -lastmod: '2017-09-03T22:22:22' -expirydate: '2025-12-31T23:59:59' ---- -Page With Date, PublishDate and LastMod` + for _, dateHandler := range []string{":filename", ":fileModTime"} { + t.Run(fmt.Sprintf("dateHandler=%q", dateHandler), func(t *testing.T) { + assrt := require.New(t) + cfg, fs := newTestCfg() - p_DP_ME = `--- -title: Simple -date: '2013-10-15T06:16:13' -publishdate: '1969-01-10T09:17:42' -modified: '2018-01-24T12:21:39' -expirydate: '2025-12-31T23:59:59' + pageTemplate := ` --- -Page With Date, PublishDate and LastMod` - - p_DPLME = `--- -title: Simple -date: '2013-10-15T06:16:13' -publishdate: '1969-01-10T09:17:42' -lastmod: '2017-09-03T22:22:22' -modified: '2018-01-24T12:21:39' -expirydate: '2025-12-31T23:59:59' +title: Page +weight: %d +lastMod: 2018-02-28 +%s --- -Page With Date, PublishDate and LastMod` +Content +` - emptyFM = `--- + cfg.Set("frontmatter", map[string]interface{}{ + "date": []string{dateHandler, "date"}, + }) ---- -Page With empty front matter` + c1 := filepath.Join("content", "section", "2012-02-21-noslug.md") + c2 := filepath.Join("content", "section", "2012-02-22-slug.md") - zero_FM = "Page With empty front matter" -) + writeSource(t, fs, c1, fmt.Sprintf(pageTemplate, 1, "")) + writeSource(t, fs, c2, fmt.Sprintf(pageTemplate, 2, "slug: aslug")) -func TestMetadataDates(t *testing.T) { - t.Parallel() - var tests = []struct { - text string - filename string - modFallback bool - expDate string - expPub string - expLast string - expMod string - expExp string - }{ - // The three columns on the left are the test case inputs: - // page content: The name indicates which dates are set in the front matter, - // (D)ate, (P)ublishDate, (L)astModified - // (M)odified, (E)xpiryDate. So, for example, - // p__PL__ is content with PublishDate and LastModified - // specified in the front matter. - // file path: For when we start deriving metadata from it - // modFallback: Whether or not useModTimeAsFallback is enabled. - // - // The single character columns on the right are the expected outputs - // for each metadata date given by the column heading. - // Since each date type (D/P/L/M/E) in the input is always set - // to the same value (the constants referenced in these columns), it - // is easy to visualize and test which input date gets copied to which - // output date fields. "s" signifies the file's filesystem time stamp, - // "x" signifies a nil value, and "o" the "zero date". - // - // ------- inputs --------|--- outputs ---| - //content filename modfb? D P L M E - {p_D____, "test.md", false, D, D, D, x, x}, // date copied across - {p_D____, "testy.md", true, D, D, D, x, x}, - {p__P___, "test.md", false, P, P, P, x, x}, // pubdate copied across - {p__P___, "testy.md", true, P, P, P, x, x}, - {p_DP___, "test.md", false, D, P, D, x, x}, // date -> lastMod - {p_DP___, "testy.md", true, D, P, D, x, x}, - {p__PL__, "test.md", false, P, P, L, x, x}, // pub -> date overrides lastMod -> date code (inconsistent?) - {p__PL__, "testy.md", true, P, P, L, x, x}, - {p_DPL__, "test.md", false, D, P, L, x, x}, // three dates - {p_DPL__, "testy.md", true, D, P, L, x, x}, - {p_DPL_E, "testy.md", true, D, P, L, x, E}, // lastMod NOT copied to mod (inconsistent?) - {p_DP_ME, "testy.md", true, D, P, M, M, E}, // mod copied to lastMod - {p_DPLME, "testy.md", true, D, P, L, M, E}, // all dates - - {emptyFM, "test.md", false, o, o, o, x, x}, // 3 year-one dates, 2 empty dates - {zero_FM, "test.md", false, o, o, o, x, x}, - {emptyFM, "testy.md", true, s, o, s, x, x}, // 2 filesys, 1 year-one, 2 empty - {zero_FM, "testy.md", true, s, o, s, x, x}, - } + c1fi, err := fs.Source.Stat(c1) + assrt.NoError(err) + c2fi, err := fs.Source.Stat(c2) + assrt.NoError(err) - for i, test := range tests { - s := newTestSite(t) - s.Cfg.Set("useModTimeAsFallback", test.modFallback) - fs := hugofs.NewMem(s.Cfg) + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - writeToFs(t, fs.Source, test.filename, test.text) - file, err := fs.Source.Open(test.filename) - if err != nil { - t.Fatal("failed to write test file to test filesystem") - } - fi, _ := fs.Source.Stat(test.filename) + assrt.Len(s.RegularPages, 2) - sp := source.NewSourceSpec(s.Cfg, fs) - p := s.newPageFromFile(newFileInfo(sp, "", test.filename, fi, bundleNot)) - p.ReadFrom(file) + noSlug := s.RegularPages[0] + slug := s.RegularPages[1] - // check Page Variables - checkDate(t, i+1, "Date", p.Date, test.expDate, fi) - checkDate(t, i+1, "PubDate", p.PublishDate, test.expPub, fi) - checkDate(t, i+1, "LastMod", p.Lastmod, test.expLast, fi) - checkDate(t, i+1, "LastMod", p.ExpiryDate, test.expExp, fi) + assrt.Equal(28, noSlug.Lastmod.Day()) - // check Page Params - checkDate(t, i+1, "param date", cast.ToTime(p.params["date"]), test.expDate, fi) - checkDate(t, i+1, "param publishdate", cast.ToTime(p.params["publishdate"]), test.expPub, fi) - checkDate(t, i+1, "param modified", cast.ToTime(p.params["modified"]), test.expMod, fi) - checkDate(t, i+1, "param expirydate", cast.ToTime(p.params["expirydate"]), test.expExp, fi) - } -} + switch strings.ToLower(dateHandler) { + case ":filename": + assrt.False(noSlug.Date.IsZero()) + assrt.False(slug.Date.IsZero()) + assrt.Equal(2012, noSlug.Date.Year()) + assrt.Equal(2012, slug.Date.Year()) + assrt.Equal("noslug", noSlug.Slug) + assrt.Equal("aslug", slug.Slug) + case ":filemodtime": + assrt.Equal(c1fi.ModTime().Year(), noSlug.Date.Year()) + assrt.Equal(c2fi.ModTime().Year(), slug.Date.Year()) + fallthrough + default: + assrt.Equal("", noSlug.Slug) + assrt.Equal("aslug", slug.Slug) -func checkDate(t *testing.T, testId int, dateType string, given time.Time, expected string, fi os.FileInfo) { - var expectedTime time.Time - if expected == s { - expectedTime = fi.ModTime() - } else if expected != x { - expectedTime = parseTime(expected, t) - } - - if given != expectedTime { - t.Errorf("test %d, %s is: %s. Expected: %s", testId, dateType, given, expectedTime) + } + }) } -} -func parseTime(s string, t *testing.T) time.Time { - time, err := time.Parse(time.RFC3339, s) - if err != nil { - t.Fatalf("bad test data: failed to parse date: '%s'", s) - } - return time } func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { diff --git a/hugolib/pagemeta/page_frontmatter.go b/hugolib/pagemeta/page_frontmatter.go new file mode 100644 index 00000000000..3bba191ff73 --- /dev/null +++ b/hugolib/pagemeta/page_frontmatter.go @@ -0,0 +1,392 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pagemeta + +import ( + "io/ioutil" + "log" + "os" + "strings" + "time" + + "github.com/gohugoio/hugo/helpers" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/cast" + jww "github.com/spf13/jwalterweatherman" +) + +// FrontMatterHandler maps front matter into Page fields and .Params. +// Note that we currently have only extracted the date logic. +type FrontMatterHandler struct { + fmConfig frontmatterConfig + + dateHandler frontMatterFieldHandler + lastModHandler frontMatterFieldHandler + publishDateHandler frontMatterFieldHandler + expiryDateHandler frontMatterFieldHandler + + // A map of all date keys configured, including any custom. + allDateKeys map[string]bool + + logger *jww.Notepad +} + +// FrontMatterDescriptor describes how to handle front matter for a given Page. +// It has pointers to values in the receiving page which gets updated. +type FrontMatterDescriptor struct { + + // This the Page's front matter. + Frontmatter map[string]interface{} + + // This is the Page's base filename, e.g. page.md. + BaseFilename string + + // The content file's mod time. + ModTime time.Time + + // The below are pointers to values on Page and will be modified. + + // This is the Page's params. + Params map[string]interface{} + + // This is the Page's dates. + Dates *PageDates + + // This is the Page's Slug etc. + PageURLs *URLPath +} + +var ( + dateFieldAliases = map[string][]string{ + fmDate: []string{}, + fmLastmod: []string{"modified"}, + fmPubDate: []string{"pubdate", "published"}, + fmExpiryDate: []string{"unpublishdate"}, + } +) + +// HandleDates updates all the dates given the current configuration and the +// supplied front matter params. Note that this requires all lower-case keys +// in the params map. +func (f FrontMatterHandler) HandleDates(d *FrontMatterDescriptor) error { + if d.Dates == nil { + panic("missing dates") + } + + if f.dateHandler == nil { + panic("missing date handler") + } + + if _, err := f.dateHandler(d); err != nil { + return err + } + + if _, err := f.lastModHandler(d); err != nil { + return err + } + + if _, err := f.publishDateHandler(d); err != nil { + return err + } + + if _, err := f.expiryDateHandler(d); err != nil { + return err + } + + return nil +} + +// IsDateKey returns whether the given front matter key is considered a date by the current +// configuration. +func (f FrontMatterHandler) IsDateKey(key string) bool { + return f.allDateKeys[key] +} + +// A Zero date is a signal that the name can not be parsed. +// This follows the format as outlined in Jekyll, https://jekyllrb.com/docs/posts/: +// "Where YEAR is a four-digit number, MONTH and DAY are both two-digit numbers" +func dateAndSlugFromBaseFilename(name string) (time.Time, string) { + withoutExt, _ := helpers.FileAndExt(name) + + if len(withoutExt) < 10 { + // This can not be a date. + return time.Time{}, "" + } + + // Note: Hugo currently have no custom timezone support. + // We will have to revisit this when that is in place. + d, err := time.Parse("2006-01-02", withoutExt[:10]) + if err != nil { + return time.Time{}, "" + } + + // Be a little lenient with the format here. + slug := strings.Trim(withoutExt[10:], " -_") + + return d, slug +} + +type frontMatterFieldHandler func(d *FrontMatterDescriptor) (bool, error) + +func (f FrontMatterHandler) newChainedFrontMatterFieldHandler(handlers ...frontMatterFieldHandler) frontMatterFieldHandler { + return func(d *FrontMatterDescriptor) (bool, error) { + for _, h := range handlers { + // First successful handler wins. + success, err := h(d) + if err != nil { + f.logger.ERROR.Println(err) + } else if success { + return true, nil + } + } + return false, nil + } +} + +type frontmatterConfig struct { + date []string + lastmod []string + publishDate []string + expiryDate []string +} + +const ( + // These are all the date handler identifiers + // All identifiers not starting with a ":" maps to a front matter parameter. + fmDate = "date" + fmPubDate = "publishdate" + fmLastmod = "lastmod" + fmExpiryDate = "expirydate" + + // Gets date from filename, e.g 218-02-22-mypage.md + fmFilename = ":filename" + + // Gets date from file OS mod time. + fmModTime = ":filemodtime" +) + +// This is the config you get when doing nothing. +func newDefaultFrontmatterConfig() frontmatterConfig { + return frontmatterConfig{ + date: []string{fmDate, fmPubDate, fmLastmod}, + lastmod: []string{fmLastmod, fmDate, fmPubDate}, + publishDate: []string{fmPubDate, fmDate}, + expiryDate: []string{fmExpiryDate}, + } +} + +func newFrontmatterConfig(cfg config.Provider) (frontmatterConfig, error) { + c := newDefaultFrontmatterConfig() + if cfg.IsSet("frontmatter") { + fm := cfg.GetStringMap("frontmatter") + if fm != nil { + for k, v := range fm { + loki := strings.ToLower(k) + switch loki { + case fmDate: + c.date = toLowerSlice(v) + case fmPubDate: + c.publishDate = toLowerSlice(v) + case fmLastmod: + c.lastmod = toLowerSlice(v) + case fmExpiryDate: + c.expiryDate = toLowerSlice(v) + } + } + } + } + + c.date = addDateFieldAliases(c.date) + c.publishDate = addDateFieldAliases(c.publishDate) + c.lastmod = addDateFieldAliases(c.lastmod) + c.expiryDate = addDateFieldAliases(c.expiryDate) + + return c, nil +} + +func addDateFieldAliases(values []string) []string { + var complete []string + + for _, v := range values { + complete = append(complete, v) + if aliases, found := dateFieldAliases[v]; found { + complete = append(complete, aliases...) + } + } + return helpers.UniqueStrings(complete) +} + +func toLowerSlice(in interface{}) []string { + out := cast.ToStringSlice(in) + for i := 0; i < len(out); i++ { + out[i] = strings.ToLower(out[i]) + } + + return out +} + +// NewFrontmatterHandler creates a new FrontMatterHandler with the given logger and configuration. +// If no logger is provided, one will be created. +func NewFrontmatterHandler(logger *jww.Notepad, cfg config.Provider) (FrontMatterHandler, error) { + + if logger == nil { + logger = jww.NewNotepad(jww.LevelWarn, jww.LevelWarn, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + } + + frontMatterConfig, err := newFrontmatterConfig(cfg) + if err != nil { + return FrontMatterHandler{}, err + } + + allDateKeys := make(map[string]bool) + addKeys := func(vals []string) { + for _, k := range vals { + if !strings.HasPrefix(k, ":") { + allDateKeys[k] = true + } + } + } + + addKeys(frontMatterConfig.date) + addKeys(frontMatterConfig.expiryDate) + addKeys(frontMatterConfig.lastmod) + addKeys(frontMatterConfig.publishDate) + + f := FrontMatterHandler{logger: logger, fmConfig: frontMatterConfig, allDateKeys: allDateKeys} + + if err := f.createHandlers(); err != nil { + return f, err + } + + return f, nil +} + +func (f *FrontMatterHandler) createHandlers() error { + var err error + + if f.dateHandler, err = f.createDateHandler(f.fmConfig.date, + func(d *FrontMatterDescriptor, t time.Time) { + d.Dates.Date = t + setParamIfNotSet(fmDate, t, d) + }); err != nil { + return err + } + + if f.lastModHandler, err = f.createDateHandler(f.fmConfig.lastmod, + func(d *FrontMatterDescriptor, t time.Time) { + setParamIfNotSet(fmLastmod, t, d) + d.Dates.Lastmod = t + }); err != nil { + return err + } + + if f.publishDateHandler, err = f.createDateHandler(f.fmConfig.publishDate, + func(d *FrontMatterDescriptor, t time.Time) { + setParamIfNotSet(fmPubDate, t, d) + d.Dates.PublishDate = t + }); err != nil { + return err + } + + if f.expiryDateHandler, err = f.createDateHandler(f.fmConfig.expiryDate, + func(d *FrontMatterDescriptor, t time.Time) { + setParamIfNotSet(fmExpiryDate, t, d) + d.Dates.ExpiryDate = t + }); err != nil { + return err + } + + return nil +} + +func setParamIfNotSet(key string, value interface{}, d *FrontMatterDescriptor) { + if _, found := d.Params[key]; found { + return + } + d.Params[key] = value +} + +func (f FrontMatterHandler) createDateHandler(identifiers []string, setter func(d *FrontMatterDescriptor, t time.Time)) (frontMatterFieldHandler, error) { + var h *frontmatterFieldHandlers + var handlers []frontMatterFieldHandler + + for _, identifier := range identifiers { + switch identifier { + case fmFilename: + handlers = append(handlers, h.newDateFilenameHandler(setter)) + case fmModTime: + handlers = append(handlers, h.newDateModTimeHandler(setter)) + default: + handlers = append(handlers, h.newDateFieldHandler(identifier, setter)) + } + } + + return f.newChainedFrontMatterFieldHandler(handlers...), nil + +} + +type frontmatterFieldHandlers int + +func (f *frontmatterFieldHandlers) newDateFieldHandler(key string, setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler { + return func(d *FrontMatterDescriptor) (bool, error) { + v, found := d.Frontmatter[key] + + if !found { + return false, nil + } + + date, err := cast.ToTimeE(v) + if err != nil { + return false, nil + } + + // We map several date keys to one, so, for example, + // "expirydate", "unpublishdate" will all set .ExpiryDate (first found). + setter(d, date) + + // This is the params key as set in front matter. + d.Params[key] = date + + return true, nil + } +} + +func (f *frontmatterFieldHandlers) newDateFilenameHandler(setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler { + return func(d *FrontMatterDescriptor) (bool, error) { + date, slug := dateAndSlugFromBaseFilename(d.BaseFilename) + if date.IsZero() { + return false, nil + } + + setter(d, date) + + if _, found := d.Frontmatter["slug"]; !found { + // Use slug from filename + d.PageURLs.Slug = slug + } + + return true, nil + } +} + +func (f *frontmatterFieldHandlers) newDateModTimeHandler(setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler { + return func(d *FrontMatterDescriptor) (bool, error) { + if d.ModTime.IsZero() { + return false, nil + } + setter(d, d.ModTime) + return true, nil + } +} diff --git a/hugolib/pagemeta/page_frontmatter_test.go b/hugolib/pagemeta/page_frontmatter_test.go new file mode 100644 index 00000000000..e2ba55371d5 --- /dev/null +++ b/hugolib/pagemeta/page_frontmatter_test.go @@ -0,0 +1,350 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pagemeta + +import ( + "fmt" + "testing" + "time" + + "github.com/spf13/viper" + + "github.com/stretchr/testify/require" +) + +func TestDateAndSlugFromBaseFilename(t *testing.T) { + + t.Parallel() + + assert := require.New(t) + + tests := []struct { + name string + date string + slug string + }{ + {"page.md", "0001-01-01", ""}, + {"2012-09-12-page.md", "2012-09-12", "page"}, + {"2018-02-28-page.md", "2018-02-28", "page"}, + {"2018-02-28_page.md", "2018-02-28", "page"}, + {"2018-02-28 page.md", "2018-02-28", "page"}, + {"2018-02-28page.md", "2018-02-28", "page"}, + {"2018-02-28-.md", "2018-02-28", ""}, + {"2018-02-28-.md", "2018-02-28", ""}, + {"2018-02-28.md", "2018-02-28", ""}, + {"2018-02-28-page", "2018-02-28", "page"}, + {"2012-9-12-page.md", "0001-01-01", ""}, + {"asdfasdf.md", "0001-01-01", ""}, + } + + for i, test := range tests { + expectedDate, err := time.Parse("2006-01-02", test.date) + assert.NoError(err) + + errMsg := fmt.Sprintf("Test %d", i) + gotDate, gotSlug := dateAndSlugFromBaseFilename(test.name) + + assert.Equal(expectedDate, gotDate, errMsg) + assert.Equal(test.slug, gotSlug, errMsg) + + } +} + +func newTestFd() *FrontMatterDescriptor { + return &FrontMatterDescriptor{ + Frontmatter: make(map[string]interface{}), + Params: make(map[string]interface{}), + Dates: &PageDates{}, + PageURLs: &URLPath{}, + } +} + +func TestFrontMatterNewConfig(t *testing.T) { + assert := require.New(t) + + cfg := viper.New() + + cfg.Set("frontmatter", map[string]interface{}{ + "date": []string{"publishDate", "LastMod"}, + "Lastmod": []string{"publishDate"}, + "expiryDate": []string{"lastMod"}, + "publishDate": []string{"date"}, + }) + + fc, err := newFrontmatterConfig(cfg) + assert.NoError(err) + assert.Equal([]string{"publishdate", "pubdate", "published", "lastmod", "modified"}, fc.date) + assert.Equal([]string{"publishdate", "pubdate", "published"}, fc.lastmod) + assert.Equal([]string{"lastmod", "modified"}, fc.expiryDate) + assert.Equal([]string{"date"}, fc.publishDate) + + // Default + cfg = viper.New() + fc, err = newFrontmatterConfig(cfg) + assert.NoError(err) + assert.Equal([]string{"date", "publishdate", "pubdate", "published", "lastmod", "modified"}, fc.date) + assert.Equal([]string{"lastmod", "modified", "date", "publishdate", "pubdate", "published"}, fc.lastmod) + assert.Equal([]string{"expirydate", "unpublishdate"}, fc.expiryDate) + assert.Equal([]string{"publishdate", "pubdate", "published", "date"}, fc.publishDate) + +} + +func TestFrontMatterDatesFilenameModTime(t *testing.T) { + assert := require.New(t) + + cfg := viper.New() + + cfg.Set("frontmatter", map[string]interface{}{ + "date": []string{":fileModTime", "date"}, + }) + + handler, err := NewFrontmatterHandler(nil, cfg) + assert.NoError(err) + + d1, _ := time.Parse("2006-01-02", "2018-02-01") + d2, _ := time.Parse("2006-01-02", "2018-02-02") + + d := newTestFd() + d.ModTime = d1 + d.Frontmatter["date"] = d2 + assert.NoError(handler.HandleDates(d)) + assert.Equal(d1, d.Dates.Date) + assert.Equal(d2, d.Params["date"]) + + d = newTestFd() + d.Frontmatter["date"] = d2 + assert.NoError(handler.HandleDates(d)) + assert.Equal(d2, d.Dates.Date) + assert.Equal(d2, d.Params["date"]) + +} + +func TestFrontMatterDatesFilename(t *testing.T) { + assert := require.New(t) + + cfg := viper.New() + + cfg.Set("frontmatter", map[string]interface{}{ + "date": []string{":filename", "date"}, + }) + + handler, err := NewFrontmatterHandler(nil, cfg) + assert.NoError(err) + + d1, _ := time.Parse("2006-01-02", "2018-02-01") + d2, _ := time.Parse("2006-01-02", "2018-02-02") + + d := newTestFd() + d.BaseFilename = "2018-02-01-page.md" + d.Frontmatter["date"] = d2 + assert.NoError(handler.HandleDates(d)) + assert.Equal(d1, d.Dates.Date) + assert.Equal(d2, d.Params["date"]) + + d = newTestFd() + d.Frontmatter["date"] = d2 + assert.NoError(handler.HandleDates(d)) + assert.Equal(d2, d.Dates.Date) + assert.Equal(d2, d.Params["date"]) +} + +func TestFrontMatterDatesCustomConfig(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + cfg := viper.New() + cfg.Set("frontmatter", map[string]interface{}{ + "date": "mydate", + "lastmod": "publishdate", + "publishdate": "publishdate", + }) + + handler, err := NewFrontmatterHandler(nil, cfg) + assert.NoError(err) + + testDate, err := time.Parse("2006-01-02", "2018-02-01") + assert.NoError(err) + + d := newTestFd() + d.Frontmatter["mydate"] = testDate + testDate = testDate.Add(24 * time.Hour) + d.Frontmatter["date"] = testDate + testDate = testDate.Add(24 * time.Hour) + d.Frontmatter["lastmod"] = testDate + testDate = testDate.Add(24 * time.Hour) + d.Frontmatter["publishdate"] = testDate + testDate = testDate.Add(24 * time.Hour) + d.Frontmatter["expirydate"] = testDate + + assert.NoError(handler.HandleDates(d)) + + assert.Equal(1, d.Dates.Date.Day()) + assert.Equal(4, d.Dates.Lastmod.Day()) + assert.Equal(4, d.Dates.PublishDate.Day()) + assert.Equal(5, d.Dates.ExpiryDate.Day()) + + assert.False(handler.IsDateKey("date")) // This looks odd, but is configured like this. + assert.True(handler.IsDateKey("mydate")) + assert.True(handler.IsDateKey("publishdate")) + assert.True(handler.IsDateKey("pubdate")) + +} + +func TestFrontMatterDatesDefaultConfig(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + cfg := viper.New() + + handler, err := NewFrontmatterHandler(nil, cfg) + assert.NoError(err) + + testDate, err := time.Parse("2006-01-02", "2018-02-01") + assert.NoError(err) + + sentinel := (time.Time{}).Add(1 * time.Hour) + zero := time.Time{} + + testDate = testDate.Add(24 * time.Hour) + + counter := 0 + + // See http://www.imdb.com/title/tt0133093/ + for _, lastModKey := range []string{"lastmod", "modified"} { + testDate = testDate.Add(24 * time.Hour) + for _, lastModDate := range []time.Time{testDate, sentinel} { + for _, dateKey := range []string{"date"} { + testDate = testDate.Add(24 * time.Hour) + for _, dateDate := range []time.Time{testDate, sentinel} { + for _, pubDateKey := range []string{"publishdate", "pubdate", "published"} { + testDate = testDate.Add(24 * time.Hour) + for _, pubDateDate := range []time.Time{testDate, sentinel} { + for _, expiryDateKey := range []string{"expirydate", "unpublishdate"} { + counter++ + + testDate = testDate.Add(24 * time.Hour) + for _, expiryDateDate := range []time.Time{testDate, sentinel} { + d := newTestFd() + var ( + expLastMod, expDate, expPubDate, expExiryDate = zero, zero, zero, zero + expParamLastModKey, expParamDateKey, expParamPubDateKey, expParamExpiryDateKey string + ) + + if dateDate != sentinel { + d.Frontmatter[dateKey] = dateDate + expDate = dateDate + expPubDate = dateDate + expParamDateKey = dateKey + } + + if pubDateDate != sentinel { + d.Frontmatter[pubDateKey] = pubDateDate + expPubDate = pubDateDate + if expDate.IsZero() { + expDate = expPubDate + } + expParamPubDateKey = pubDateKey + } + + if lastModDate != sentinel { + d.Frontmatter[lastModKey] = lastModDate + expLastMod = lastModDate + + if expDate.IsZero() { + expDate = lastModDate + } + expParamLastModKey = lastModKey + } + + if expiryDateDate != sentinel { + d.Frontmatter[expiryDateKey] = expiryDateDate + expExiryDate = expiryDateDate + expParamExpiryDateKey = expiryDateKey + } + + if expLastMod.IsZero() { + expLastMod = expDate + } + + assert.True(handler.IsDateKey(dateKey)) + assert.True(handler.IsDateKey(pubDateKey)) + assert.True(handler.IsDateKey(lastModKey)) + assert.True(handler.IsDateKey(expiryDateKey)) + + assert.NoError(handler.HandleDates(d)) + + message := fmt.Sprintf("Test %d", counter) + + assertFrontMatterDateInParams("date", assert, message, d, expDate, d.Dates.Date, expParamDateKey) + assertFrontMatterDateInParams("lastMod", assert, message, d, expLastMod, d.Dates.Lastmod, expParamLastModKey) + assertFrontMatterDateInParams("pubishDate", assert, message, d, expPubDate, d.Dates.PublishDate, expParamPubDateKey) + assertFrontMatterDateInParams("expiryDate", assert, message, d, expExiryDate, d.Dates.ExpiryDate, expParamExpiryDateKey) + } + } + } + } + } + } + } + } +} + +func assertFrontMatterDateInParams(name string, assert *require.Assertions, message string, + d *FrontMatterDescriptor, expected time.Time, got time.Time, dateField string) { + + if dateField != "" { + param, found := d.Params[dateField] + + if found && param.(time.Time).IsZero() { + assert.Fail("Zero time in params", dateField, message) + } + + message = fmt.Sprintf("[%s][%s]\nFound: %t\nExpected: %v (%t)\nParam: %v\nParams: %v\nFront matter: %v", + message, dateField, found, expected, expected.IsZero(), param, d.Params, d.Frontmatter) + + assert.True(found != expected.IsZero(), message) + + if found { + if expected != param { + assert.Failf("Params check failed", "[%s] Expected:\n%q\nGot:\n%q", dateField, expected, param) + } + } + } + + if expected != got { + assert.Failf("Field check", "[%s][%s] Expected\n%q\ngot\n%q\nFront matter: %v", message, name, expected, got, d.Frontmatter) + } + + assert.Equal(expected, got) +} + +func TestFrontMatterDateFieldHandler(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + handlers := new(frontmatterFieldHandlers) + + fd := newTestFd() + d, _ := time.Parse("2006-01-02", "2018-02-01") + fd.Frontmatter["date"] = d + h := handlers.newDateFieldHandler("date", func(d *FrontMatterDescriptor, t time.Time) { d.Dates.Date = t }) + + handled, err := h(fd) + assert.True(handled) + assert.NoError(err) + assert.Equal(d, fd.Dates.Date) +} diff --git a/hugolib/pagemeta/pagemeta.go b/hugolib/pagemeta/pagemeta.go new file mode 100644 index 00000000000..93dc9a12f0b --- /dev/null +++ b/hugolib/pagemeta/pagemeta.go @@ -0,0 +1,32 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pagemeta + +import ( + "time" +) + +type URLPath struct { + URL string + Permalink string + Slug string + Section string +} + +type PageDates struct { + Date time.Time + Lastmod time.Time + PublishDate time.Time + ExpiryDate time.Time +} diff --git a/hugolib/site.go b/hugolib/site.go index 95cd0a23edf..48771410d96 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -42,6 +42,7 @@ import ( bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugolib/pagemeta" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/related" @@ -121,6 +122,9 @@ type Site struct { outputFormatsConfig output.Formats mediaTypesConfig media.Types + // How to handle page front matter. + frontmatterHandler pagemeta.FrontMatterHandler + // We render each site for all the relevant output formats in serial with // this rendering context pointing to the current one. rc *siteRenderingContext @@ -177,6 +181,7 @@ func (s *Site) reset() *Site { relatedDocsHandler: newSearchIndexHandler(s.relatedDocsHandler.cfg), outputFormats: s.outputFormats, outputFormatsConfig: s.outputFormatsConfig, + frontmatterHandler: s.frontmatterHandler, mediaTypesConfig: s.mediaTypesConfig, resourceSpec: s.resourceSpec, Language: s.Language, @@ -248,6 +253,11 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { titleFunc := helpers.GetTitleFunc(cfg.Language.GetString("titleCaseStyle")) + frontMatterHandler, err := pagemeta.NewFrontmatterHandler(cfg.Logger, cfg.Cfg) + if err != nil { + return nil, err + } + s := &Site{ PageCollections: c, layoutHandler: output.NewLayoutHandler(cfg.Cfg.GetString("themesDir") != ""), @@ -258,6 +268,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { outputFormats: outputFormats, outputFormatsConfig: siteOutputFormatsConfig, mediaTypesConfig: siteMediaTypesConfig, + frontmatterHandler: frontMatterHandler, } s.Info = newSiteInfo(siteBuilderCfg{s: s, pageCollections: c, language: s.Language})