Minor fixes + sitemap generation (#631)

* closes #621

* closes #599

* add sitemap generator, closes #491

* patch release

Project.toml

@@ -1,7 +1,7 @@
 name = "Franklin"
 uuid = "713c75ef-9fc9-4b05-94a9-213340da978e"
 authors = ["Thibaut Lienart <tlienart@me.com>"]
-version = "0.10.1"
+version = "0.10.2"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

src/Franklin.jl

@@ -86,8 +86,9 @@ const FD_ENV = LittleDict(
     :SHOW_WARNINGS => true,     # franklin-specific warnings
     )
 
-# keep track of pages which need to be re-evaluated after the full-pass to ensure that
-# their h-fun are working with the fully-defined scope (e.g. if need tags)
+# keep track of pages which need to be re-evaluated after the full-pass
+# to ensure that their h-fun are working with the fully-defined scope
+# (e.g. if need list of all tags)
 const DELAYED = Set{String}()
 
 """Dict to keep track of languages and how comments are indicated and their extensions. This is relevant to allow hiding lines of code. """
@@ -203,6 +204,7 @@ include("converter/html/prerender.jl")
 
 # FILE AND DIR MANAGEMENT
 include("manager/rss_generator.jl")
+include("manager/sitemap_generator.jl")
 include("manager/write_page.jl")
 include("manager/dir_utils.jl")
 include("manager/file_utils.jl")

src/converter/html/functions.jl

@@ -243,6 +243,8 @@ $(SIGNATURES)
 H-Function of the form `{{redirect /addr/blah.html}}`.
 """
 function hfun_redirect(params::Vector{String})::String
+    # don't put those on the sitemap
+    set_var!(LOCAL_VARS, "sitemap_exclude", true)
     if length(params) != 1
         throw(HTMLFunctionError(
                 "I found an {{redirect ...}} block and expected a single " *
@@ -333,3 +335,40 @@ function hfun_paginate(params::Vector{String})::String
     # return a token which will be processed at the convert_and_write stage.
     return PAGINATE
 end
+
+
+"""
+    hfun_sitemap_opts
+
+Called with `{{sitemap_opts monthly 0.5}}`. It is assumed this is called only
+on raw html pages (e.g. custom landing page).
+
+## Example usage
+
+* `{{sitemap_opts exclude}}`
+* `{{sitemap_opts monthly 0.5}}`
+"""
+function hfun_sitemap_opts(params::Vector{String})::String
+    # Check arguments
+    if length(params) == 1 && lowercase(params[1]) != "exclude"
+        throw(HTMLFunctionError(
+                "I found an {{sitemap_opts xxx}} block with 1 arg and " *
+                "that is only allowed if the arg is 'exclude'. Verify."))
+    elseif length(params) != 2
+        throw(HTMLFunctionError(
+                "I found an {{sitemap_opts ...}} block and expected 2 args: " *
+                "the changefreq and the priority. " *
+                "I see $(length(params)) arguments instead. Verify."))
+    end
+    key = url_curpage()
+    if params[1] == "exclude"
+        delete!(SITEMAP_DICT, key)
+        return ""
+    end
+    changefreq = params[1]
+    priority = params[2]
+    fp = joinpath(path(:folder), locvar(:fd_rpath))
+    lastmod = Date(unix2datetime(stat(fp).mtime))
+    SITEMAP_DICT[key] = SMOpts(lastmod, changefreq, priority)
+    return ""
+end

src/converter/markdown/blocks.jl

@@ -118,7 +118,7 @@ function convert_header(β::OCBlock, lxdefs::Vector{LxDef})::String
     hk    = lowercase(string(β.name)) # h1, h2, ...
     title = convert_md(content(β), lxdefs;
                        isrecursive=true, has_mddefs=false)
-    rstitle = refstring(title)
+    rstitle = refstring(html_unescape(title))
     # check if the header has appeared before and if so suggest
     # an altered refstring; if that altered refstring also exist
     # (pathological case, see #241), then extend it with a letter

src/converter/markdown/mddefs.jl

@@ -72,7 +72,7 @@ function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool,
     # if in config file, update `GLOBAL_VARS` and return
     rpath = splitext(locvar(:fd_rpath))[1]
     if isconfig
-        set_vars!(GLOBAL_VARS, assignments)
+        set_vars!(GLOBAL_VARS, assignments, isglobal=true)
         return nothing
     end
 

src/eval/codeblock.jl

@@ -36,6 +36,9 @@ The code should be reevaluated if any of following flags are true:
 1. the output is missings
 """
 function should_eval(code::AS, rpath::AS)
+    # 0. the page is currently delayed, skip evals
+    FD_ENV[:SOURCE] in DELAYED && return false
+
     # 1. global setting forcing all pages to reeval
     FD_ENV[:FORCE_REEVAL] && return true
 

src/manager/file_utils.jl

@@ -8,28 +8,13 @@ The keyword `init` is used internally to distinguish between the first call
 where only structural variables are considered (e.g. controlling folder
 structure).
 """
-function process_config(; init::Bool=false)::Nothing
+function process_config()::Nothing
     FD_ENV[:SOURCE] = "config.md"
-    if init
-        # initially the paths variable aren't set, try to find a config.md
-        # and read definitions in it; in particular folder_structure.
-        config_path_v1 = joinpath(FOLDER_PATH[], "src", "config.md")
-        config_path_v2 = joinpath(FOLDER_PATH[], "config.md")
-        if isfile(config_path_v2)
-            convert_md(read(config_path_v2, String); isconfig=true)
-        elseif isfile(config_path_v1)
-            convert_md(read(config_path_v1, String); isconfig=true)
-        else
-            config_warn()
-        end
+    config_path = joinpath(FOLDER_PATH[], "config.md")
+    if isfile(config_path)
+        convert_md(read(config_path, String); isconfig=true)
     else
-        dir = path(:folder)
-        config_path = joinpath(dir, "config.md")
-        if isfile(config_path)
-            convert_md(read(config_path, String); isconfig=true)
-        else
-            config_warn()
-        end
+        config_warn()
     end
     return nothing
 end
@@ -45,15 +30,8 @@ recommended.
 """
 function process_utils()
     FD_ENV[:SOURCE] = "utils.jl"
-    utils_path_v1 = joinpath(FOLDER_PATH[], "src", "utils.jl")
-    utils_path_v2 = joinpath(FOLDER_PATH[], "utils.jl")
-    if isfile(utils_path_v2)
-        utils = utils_path_v2
-    elseif isfile(utils_path_v1)
-        utils = utils_path_v1
-    else
-        return nothing
-    end
+    utils = joinpath(FOLDER_PATH[], "utils.jl")
+    isfile(utils) || return nothing
     # wipe / create module Utils
     newmodule("Utils")
     Base.include(Main.Utils, utils)
@@ -122,6 +100,10 @@ function process_file_err(case::Symbol, fpair::Pair{String, String},
         set_cur_rpath(joinpath(fpair...))
         set_page_env()
         raw_html  = read(inp, String)
+        # add the item *before* the conversion so that the conversion
+        # can affect the page itself with {{...}}
+        cond_add = globvar(:generate_sitemap) && FD_ENV[:FULL_PASS]
+        cond_add && add_sitemap_item(html=true)
         proc_html = convert_html(raw_html) |> postprocess_page
         write(outp, proc_html)
     else # case in (:other, :infra)

src/manager/franklin.jl

@@ -89,7 +89,7 @@ function serve(; clear::Bool=false,
 
     # check if there's a config file, if there is, check the variable
     # definitions looking at the ones that would affect overall structure etc.
-    process_config(init=true)
+    process_config()
 
     if !all(isdir, (joinpath(FOLDER_PATH[], "_layout"),
                     joinpath(FOLDER_PATH[], "_css")))
@@ -202,7 +202,7 @@ function fd_fullpass(watched_files::NamedTuple)::Int
     prepath = get(GLOBAL_VARS, "prepath", "")
     def_GLOBAL_VARS!()
     def_GLOBAL_LXDEFS!()
-    empty!(RSS_DICT)
+    empty!.((RSS_DICT, SITEMAP_DICT))
     # reinsert prepath if specified
     isempty(prepath) || (GLOBAL_VARS["prepath"] = prepath)
 
@@ -268,6 +268,8 @@ function fd_fullpass(watched_files::NamedTuple)::Int
     globvar("generate_rss") && rss_generator()
     # generate tags if appropriate
     generate_tag_pages()
+    # generate sitemap if appropriate
+    globvar("generate_sitemap") && sitemap_generator()
     # done
     FD_ENV[:FULL_PASS] = false
     # return -1 if any page failed to build, 0 otherwise

src/manager/rss_generator.jl

@@ -59,7 +59,7 @@ $SIGNATURES
 
 Create an `RSSItem` out of the provided fields defined in the page vars.
 """
-function add_rss_item()::RSSItem
+function add_rss_item()
     link  = url_curpage()
     title = jor("rss_title", "title")
     descr = jor("rss", "rss_description")
@@ -85,8 +85,9 @@ function add_rss_item()::RSSItem
         An RSS description was found but without title for page '$link'.
         """)
 
-    RSS_DICT[link] = RSSItem(title, link, descr, author,
-                             category, comments, enclosure, pubDate)
+    res = RSS_DICT[link] = RSSItem(title, link, descr, author,
+                                   category, comments, enclosure, pubDate)
+    return res
 end
 
 

src/manager/sitemap_generator.jl

@@ -0,0 +1,95 @@
+# <?xml version="1.0" encoding="utf-8" standalone="yes" ?>
+# <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+# <url>
+#      <loc>http://www.example.com/</loc>
+#      ?<lastmod>2005-01-01</lastmod>
+#      ?<changefreq>monthly</changefreq>
+#      ?<priority>0.8</priority>
+# </url>
+# </urlset>
+#
+# TODO
+# allow {{sitemap_exclude}} for a HTML page to avoid having it in sitemap
+#
+
+struct SMOpts
+    lastmod::Date        # 2005-01-01 -- format YYYY-MM-DD
+    changefreq::String   # one of...
+    priority::Float64    #
+    function SMOpts(l, c, p)
+        c = lowercase(c)
+        allowedf = ("always", "hourly", "daily", "weekly", "monthly",
+        "yearly", "never")
+        assertf = """
+        Given change frequency on $(FD_ENV[:SOURCE]) is invalid according to
+        sitemap specifications, expected one of $allowedf.
+        """
+        assertp = """
+        Given priority on $(FD_ENV[:SOURCE]) is invalid according to sitemap
+        specifications, expected a floating point number between 0 and 1.
+        """
+        @assert c in allowedf assertf
+        @assert 0 <= p <= 1 assertp
+        return new(l, c, p)
+    end
+end
+
+const SITEMAP_DICT = LittleDict{String,SMOpts}()
+
+"""
+$SIGNATURES
+
+Add an entry to `SITEMAP_DICT`.
+"""
+function add_sitemap_item(; html=false)
+    loc = url_curpage()
+    locvar(:sitemap_exclude) && return nothing
+    if !html
+        lastmod = locvar(:fd_mtime)
+        changefreq = locvar(:sitemap_changefreq)
+        priority = locvar(:sitemap_priority)
+    else
+        # use default which can be overwritten in a {{sitemap_opts ...}}
+        fp = joinpath(path(:folder), locvar(:fd_rpath))
+        lastmod = Date(unix2datetime(stat(fp).mtime))
+        changefreq = "monthly"
+        priority = 0.5
+    end
+    res = SITEMAP_DICT[loc] = SMOpts(lastmod, changefreq, priority)
+    return res
+end
+
+"""
+$SIGNATURES
+
+Generate a `sitemap.xml`, if one already exists, it will be replaced.
+"""
+function sitemap_generator()
+    dst = joinpath(path(:site), "sitemap.xml")
+    isfile(dst) && rm(dst)
+    io = IOBuffer()
+    println(io, """
+        <?xml version="1.0" encoding="utf-8" standalone="yes" ?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        """)
+    base_url = globvar(:website_url)
+    for (k, v) in SITEMAP_DICT
+        key = joinpath(escapeuri.(split(k, '/'))...)
+        loc = "<loc>$(joinpath(base_url, key))</loc>"
+        lastmod = ifelse(v.lastmod > Date(1),
+                         "<lastmod>$(v.lastmod)</lastmod>", "")
+        changefreq = "<changefreq>$(v.changefreq)</changefreq>"
+        priority = "<priority>$(v.priority)</priority>"
+        write(io, """
+            <url>
+                $loc
+                $lastmod
+                $changefreq
+                $priority
+            </url>
+            """)
+    end
+    println(io, "</urlset>")
+    write(dst, take!(io))
+    return nothing
+end

src/manager/write_page.jl

@@ -162,12 +162,16 @@ function convert_and_write(root::String, file::String, head::String,
     #   should we generate ? otherwise no
     #   are we in the full pass ? otherwise no
     #   is there a `rss` or `rss_description` ? otherwise no
-    cond_add = GLOBAL_VARS["generate_rss"].first &&  # should we generate?
-                    FD_ENV[:FULL_PASS] &&            # are we in the full pass?
+    cond_add = globvar(:generate_rss) &&   # should we generate?
+                    FD_ENV[:FULL_PASS] &&  # are we in the full pass?
                     !all(e -> isempty(locvar(e)), ("rss", "rss_description"))
     # otherwise yes
     cond_add && add_rss_item()
 
+    # Same for the sitemap
+    cond_add = globvar(:generate_sitemap) && FD_ENV[:FULL_PASS]
+    cond_add && add_sitemap_item()
+
     # adding document variables to the dictionary
     # note that some won't change and so it's not necessary to do this every
     # time but it takes negligible time to do this so ¯\_(ツ)_/¯

src/utils/misc.jl

@@ -266,3 +266,17 @@ macro delay(defun)
     end
     esc(combinedef(def))
 end
+
+# URI encoding stolen from HTTP.jl (+ simplified)
+
+# RFC3986 Unreserved Characters (and '~' Unsafe per RFC1738).
+issafe(c::Char) = c == '-' ||
+                  c == '.' ||
+                  c == '_' ||
+                  (isascii(c) && (isletter(c) || isnumeric(c)))
+
+utf8(s::AS) = (Char(c) for c in codeunits(s))
+
+escapeuri(c::Char) = string('%', uppercase(string(Int(c), base=16, pad=2)))
+escapeuri(str::AS) =
+    join(ifelse(issafe(c), c, escapeuri(Char(c))) for c in utf8(str))