Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Minor fixes + sitemap generation #631

Merged
merged 4 commits into from
Sep 27, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "Franklin"
uuid = "713c75ef-9fc9-4b05-94a9-213340da978e"
authors = ["Thibaut Lienart <tlienart@me.com>"]
version = "0.10.1"
version = "0.10.2"

[deps]
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Expand Down
6 changes: 4 additions & 2 deletions src/Franklin.jl
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,9 @@ const FD_ENV = LittleDict(
:SHOW_WARNINGS => true, # franklin-specific warnings
)

# keep track of pages which need to be re-evaluated after the full-pass to ensure that
# their h-fun are working with the fully-defined scope (e.g. if need tags)
# keep track of pages which need to be re-evaluated after the full-pass
# to ensure that their h-fun are working with the fully-defined scope
# (e.g. if need list of all tags)
const DELAYED = Set{String}()

"""Dict to keep track of languages and how comments are indicated and their extensions. This is relevant to allow hiding lines of code. """
Expand Down Expand Up @@ -203,6 +204,7 @@ include("converter/html/prerender.jl")

# FILE AND DIR MANAGEMENT
include("manager/rss_generator.jl")
include("manager/sitemap_generator.jl")
include("manager/write_page.jl")
include("manager/dir_utils.jl")
include("manager/file_utils.jl")
Expand Down
39 changes: 39 additions & 0 deletions src/converter/html/functions.jl
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ $(SIGNATURES)
H-Function of the form `{{redirect /addr/blah.html}}`.
"""
function hfun_redirect(params::Vector{String})::String
# don't put those on the sitemap
set_var!(LOCAL_VARS, "sitemap_exclude", true)
if length(params) != 1
throw(HTMLFunctionError(
"I found an {{redirect ...}} block and expected a single " *
Expand Down Expand Up @@ -333,3 +335,40 @@ function hfun_paginate(params::Vector{String})::String
# return a token which will be processed at the convert_and_write stage.
return PAGINATE
end


"""
hfun_sitemap_opts

Called with `{{sitemap_opts monthly 0.5}}`. It is assumed this is called only
on raw html pages (e.g. custom landing page).

## Example usage

* `{{sitemap_opts exclude}}`
* `{{sitemap_opts monthly 0.5}}`
"""
function hfun_sitemap_opts(params::Vector{String})::String
# Check arguments
if length(params) == 1 && lowercase(params[1]) != "exclude"
throw(HTMLFunctionError(
"I found an {{sitemap_opts xxx}} block with 1 arg and " *
"that is only allowed if the arg is 'exclude'. Verify."))
elseif length(params) != 2
throw(HTMLFunctionError(
"I found an {{sitemap_opts ...}} block and expected 2 args: " *
"the changefreq and the priority. " *
"I see $(length(params)) arguments instead. Verify."))
end
key = url_curpage()
if params[1] == "exclude"
delete!(SITEMAP_DICT, key)
return ""
end
changefreq = params[1]
priority = params[2]
fp = joinpath(path(:folder), locvar(:fd_rpath))
lastmod = Date(unix2datetime(stat(fp).mtime))
SITEMAP_DICT[key] = SMOpts(lastmod, changefreq, priority)
return ""
end
2 changes: 1 addition & 1 deletion src/converter/markdown/blocks.jl
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ function convert_header(β::OCBlock, lxdefs::Vector{LxDef})::String
hk = lowercase(string(β.name)) # h1, h2, ...
title = convert_md(content(β), lxdefs;
isrecursive=true, has_mddefs=false)
rstitle = refstring(title)
rstitle = refstring(html_unescape(title))
# check if the header has appeared before and if so suggest
# an altered refstring; if that altered refstring also exist
# (pathological case, see #241), then extend it with a letter
Expand Down
2 changes: 1 addition & 1 deletion src/converter/markdown/mddefs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool,
# if in config file, update `GLOBAL_VARS` and return
rpath = splitext(locvar(:fd_rpath))[1]
if isconfig
set_vars!(GLOBAL_VARS, assignments)
set_vars!(GLOBAL_VARS, assignments, isglobal=true)
return nothing
end

Expand Down
3 changes: 3 additions & 0 deletions src/eval/codeblock.jl
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ The code should be reevaluated if any of following flags are true:
1. the output is missings
"""
function should_eval(code::AS, rpath::AS)
# 0. the page is currently delayed, skip evals
FD_ENV[:SOURCE] in DELAYED && return false

# 1. global setting forcing all pages to reeval
FD_ENV[:FORCE_REEVAL] && return true

Expand Down
40 changes: 11 additions & 29 deletions src/manager/file_utils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -8,28 +8,13 @@ The keyword `init` is used internally to distinguish between the first call
where only structural variables are considered (e.g. controlling folder
structure).
"""
function process_config(; init::Bool=false)::Nothing
function process_config()::Nothing
FD_ENV[:SOURCE] = "config.md"
if init
# initially the paths variable aren't set, try to find a config.md
# and read definitions in it; in particular folder_structure.
config_path_v1 = joinpath(FOLDER_PATH[], "src", "config.md")
config_path_v2 = joinpath(FOLDER_PATH[], "config.md")
if isfile(config_path_v2)
convert_md(read(config_path_v2, String); isconfig=true)
elseif isfile(config_path_v1)
convert_md(read(config_path_v1, String); isconfig=true)
else
config_warn()
end
config_path = joinpath(FOLDER_PATH[], "config.md")
if isfile(config_path)
convert_md(read(config_path, String); isconfig=true)
else
dir = path(:folder)
config_path = joinpath(dir, "config.md")
if isfile(config_path)
convert_md(read(config_path, String); isconfig=true)
else
config_warn()
end
config_warn()
end
return nothing
end
Expand All @@ -45,15 +30,8 @@ recommended.
"""
function process_utils()
FD_ENV[:SOURCE] = "utils.jl"
utils_path_v1 = joinpath(FOLDER_PATH[], "src", "utils.jl")
utils_path_v2 = joinpath(FOLDER_PATH[], "utils.jl")
if isfile(utils_path_v2)
utils = utils_path_v2
elseif isfile(utils_path_v1)
utils = utils_path_v1
else
return nothing
end
utils = joinpath(FOLDER_PATH[], "utils.jl")
isfile(utils) || return nothing
# wipe / create module Utils
newmodule("Utils")
Base.include(Main.Utils, utils)
Expand Down Expand Up @@ -122,6 +100,10 @@ function process_file_err(case::Symbol, fpair::Pair{String, String},
set_cur_rpath(joinpath(fpair...))
set_page_env()
raw_html = read(inp, String)
# add the item *before* the conversion so that the conversion
# can affect the page itself with {{...}}
cond_add = globvar(:generate_sitemap) && FD_ENV[:FULL_PASS]
cond_add && add_sitemap_item(html=true)
proc_html = convert_html(raw_html) |> postprocess_page
write(outp, proc_html)
else # case in (:other, :infra)
Expand Down
6 changes: 4 additions & 2 deletions src/manager/franklin.jl
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ function serve(; clear::Bool=false,

# check if there's a config file, if there is, check the variable
# definitions looking at the ones that would affect overall structure etc.
process_config(init=true)
process_config()

if !all(isdir, (joinpath(FOLDER_PATH[], "_layout"),
joinpath(FOLDER_PATH[], "_css")))
Expand Down Expand Up @@ -202,7 +202,7 @@ function fd_fullpass(watched_files::NamedTuple)::Int
prepath = get(GLOBAL_VARS, "prepath", "")
def_GLOBAL_VARS!()
def_GLOBAL_LXDEFS!()
empty!(RSS_DICT)
empty!.((RSS_DICT, SITEMAP_DICT))
# reinsert prepath if specified
isempty(prepath) || (GLOBAL_VARS["prepath"] = prepath)

Expand Down Expand Up @@ -268,6 +268,8 @@ function fd_fullpass(watched_files::NamedTuple)::Int
globvar("generate_rss") && rss_generator()
# generate tags if appropriate
generate_tag_pages()
# generate sitemap if appropriate
globvar("generate_sitemap") && sitemap_generator()
# done
FD_ENV[:FULL_PASS] = false
# return -1 if any page failed to build, 0 otherwise
Expand Down
7 changes: 4 additions & 3 deletions src/manager/rss_generator.jl
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ $SIGNATURES

Create an `RSSItem` out of the provided fields defined in the page vars.
"""
function add_rss_item()::RSSItem
function add_rss_item()
link = url_curpage()
title = jor("rss_title", "title")
descr = jor("rss", "rss_description")
Expand All @@ -85,8 +85,9 @@ function add_rss_item()::RSSItem
An RSS description was found but without title for page '$link'.
""")

RSS_DICT[link] = RSSItem(title, link, descr, author,
category, comments, enclosure, pubDate)
res = RSS_DICT[link] = RSSItem(title, link, descr, author,
category, comments, enclosure, pubDate)
return res
end


Expand Down
95 changes: 95 additions & 0 deletions src/manager/sitemap_generator.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# <?xml version="1.0" encoding="utf-8" standalone="yes" ?>
# <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
# <url>
# <loc>http://www.example.com/</loc>
# ?<lastmod>2005-01-01</lastmod>
# ?<changefreq>monthly</changefreq>
# ?<priority>0.8</priority>
# </url>
# </urlset>
#
# TODO
# allow {{sitemap_exclude}} for a HTML page to avoid having it in sitemap
#

struct SMOpts
lastmod::Date # 2005-01-01 -- format YYYY-MM-DD
changefreq::String # one of...
priority::Float64 #
function SMOpts(l, c, p)
c = lowercase(c)
allowedf = ("always", "hourly", "daily", "weekly", "monthly",
"yearly", "never")
assertf = """
Given change frequency on $(FD_ENV[:SOURCE]) is invalid according to
sitemap specifications, expected one of $allowedf.
"""
assertp = """
Given priority on $(FD_ENV[:SOURCE]) is invalid according to sitemap
specifications, expected a floating point number between 0 and 1.
"""
@assert c in allowedf assertf
@assert 0 <= p <= 1 assertp
return new(l, c, p)
end
end

const SITEMAP_DICT = LittleDict{String,SMOpts}()

"""
$SIGNATURES

Add an entry to `SITEMAP_DICT`.
"""
function add_sitemap_item(; html=false)
loc = url_curpage()
locvar(:sitemap_exclude) && return nothing
if !html
lastmod = locvar(:fd_mtime)
changefreq = locvar(:sitemap_changefreq)
priority = locvar(:sitemap_priority)
else
# use default which can be overwritten in a {{sitemap_opts ...}}
fp = joinpath(path(:folder), locvar(:fd_rpath))
lastmod = Date(unix2datetime(stat(fp).mtime))
changefreq = "monthly"
priority = 0.5
end
res = SITEMAP_DICT[loc] = SMOpts(lastmod, changefreq, priority)
return res
end

"""
$SIGNATURES

Generate a `sitemap.xml`, if one already exists, it will be replaced.
"""
function sitemap_generator()
dst = joinpath(path(:site), "sitemap.xml")
isfile(dst) && rm(dst)
io = IOBuffer()
println(io, """
<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
""")
base_url = globvar(:website_url)
for (k, v) in SITEMAP_DICT
key = joinpath(escapeuri.(split(k, '/'))...)
loc = "<loc>$(joinpath(base_url, key))</loc>"
lastmod = ifelse(v.lastmod > Date(1),
"<lastmod>$(v.lastmod)</lastmod>", "")
changefreq = "<changefreq>$(v.changefreq)</changefreq>"
priority = "<priority>$(v.priority)</priority>"
write(io, """
<url>
$loc
$lastmod
$changefreq
$priority
</url>
""")
end
println(io, "</urlset>")
write(dst, take!(io))
return nothing
end
8 changes: 6 additions & 2 deletions src/manager/write_page.jl
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,16 @@ function convert_and_write(root::String, file::String, head::String,
# should we generate ? otherwise no
# are we in the full pass ? otherwise no
# is there a `rss` or `rss_description` ? otherwise no
cond_add = GLOBAL_VARS["generate_rss"].first && # should we generate?
FD_ENV[:FULL_PASS] && # are we in the full pass?
cond_add = globvar(:generate_rss) && # should we generate?
FD_ENV[:FULL_PASS] && # are we in the full pass?
!all(e -> isempty(locvar(e)), ("rss", "rss_description"))
# otherwise yes
cond_add && add_rss_item()

# Same for the sitemap
cond_add = globvar(:generate_sitemap) && FD_ENV[:FULL_PASS]
cond_add && add_sitemap_item()

# adding document variables to the dictionary
# note that some won't change and so it's not necessary to do this every
# time but it takes negligible time to do this so ¯\_(ツ)_/¯
Expand Down
14 changes: 14 additions & 0 deletions src/utils/misc.jl
Original file line number Diff line number Diff line change
Expand Up @@ -266,3 +266,17 @@ macro delay(defun)
end
esc(combinedef(def))
end

# URI encoding stolen from HTTP.jl (+ simplified)

# RFC3986 Unreserved Characters (and '~' Unsafe per RFC1738).
issafe(c::Char) = c == '-' ||
c == '.' ||
c == '_' ||
(isascii(c) && (isletter(c) || isnumeric(c)))

utf8(s::AS) = (Char(c) for c in codeunits(s))

escapeuri(c::Char) = string('%', uppercase(string(Int(c), base=16, pad=2)))
escapeuri(str::AS) =
join(ifelse(issafe(c), c, escapeuri(Char(c))) for c in utf8(str))
Loading