gen

#!/usr/bin/env python3

# Copyright Muxup contributors.
# Distributed under the terms of the MIT-0 license, see LICENSE for details.
# SPDX-License-Identifier: MIT-0

import argparse
import atexit
import concurrent.futures
import datetime
import html
import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
import threading
import tomllib
import urllib.parse
from dataclasses import dataclass
from typing import Any, Callable, List, Tuple, TypeVar

import mistletoe
import mistletoe.utils
import pygments.style
import pygments.token
import pygments.util
from pygments import highlight
from pygments.formatters.html import HtmlFormatter
from pygments.lexers import get_lexer_by_name as get_lexer

# All paths etc are relative to the location of this script.
os.chdir(pathlib.Path(__file__).parent)

# Data definitions. Due to the use of ProcessPoolExecutor, these should not be
# modified during script execution.
base_url = "https://muxup.com"

favicon_svg = (
    """\
<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'>\
<text y='80' font-size='110'>▨</text>\
</svg>\
""".replace('"', "%22")
    .replace("<", "%3C")
    .replace("/", "%2F")
    .replace(">", "%3E")
    .replace("#", "%23")
    .replace(" ", "%20")
)

pages_path = pathlib.Path("pages")
staged_path = pathlib.Path("staged")
local_serve_path = pathlib.Path("local_serve")
deployed_path = pathlib.Path("deployed")


# Helpers
def compile_template(
    template_str: str, template_name: str = "<template>"
) -> Callable[..., str]:
    out = []
    indent = 0
    stack = []

    def emit_line(line: str) -> None:
        out.append(f"{'    ' * indent}{line}")

    emit_line("def _render():")
    indent += 1
    emit_line("global _tpl_line")
    emit_line("out = []")

    for line_no, line in enumerate(template_str.splitlines(), start=1):
        emit_line(f"_tpl_line = {line_no}")
        if line.startswith("$"):
            pycmd = line[1:].strip()
            keyword = pycmd.partition(" ")[0]
            if keyword == "if":
                stack.append(keyword)
                emit_line(f"{pycmd}:")
                indent += 1
            elif keyword == "for":
                stack.append(keyword)
                emit_line(f"{pycmd}:")
                indent += 1
            elif keyword in ("elif", "else"):
                if stack[-1] != "if":
                    raise ValueError(
                        f"{template_name}:{line_no}: Incorrectly nested '{keyword}'"
                    )
                indent -= 1
                emit_line(f"{pycmd}:")
                indent += 1
            elif keyword in ("endif", "endfor"):
                expected = stack.pop()
                if expected != keyword[3:]:
                    raise ValueError(
                        f"{template_name}:{line_no}: "
                        f"Expected end{expected}, got {pycmd}"
                    )
                if pycmd != keyword:
                    raise ValueError(
                        f"{template_name}:{line_no}: Unexpected text after {keyword}"
                    )
                indent -= 1
            else:
                emit_line(f"{pycmd}")
            continue

        pos = 0
        while pos <= len(line):
            expr_start = line.find("{{", pos)
            if expr_start == -1:
                emit_line(f"out.append({repr(line[pos:])} '\\n')")
                break
            if expr_start != pos:
                emit_line(f"out.append({repr(line[pos:expr_start])})")
            expr_end = line.find("}}", expr_start)
            if expr_end == -1:
                raise ValueError(
                    f"{template_name}:{line_no}: Couldn't find matching }}}}"
                )
            emit_line(f"out.append(str({line[expr_start + 2 : expr_end]}))")
            pos = expr_end + 2
    if len(stack) != 0:
        raise ValueError(f"{template_name}:{line_no}: Unclosed '{stack[-1]}'")
    emit_line('return "".join(out)')
    py_code = "\n".join(out)
    compiled_code = compile(py_code, template_name, "exec")

    def wrapper(**kwargs_as_globals: Any) -> str:
        exec(compiled_code, kwargs_as_globals)
        try:
            return kwargs_as_globals["_render"]()  # type: ignore
        except Exception as e:
            tpl_line = kwargs_as_globals.get("_tpl_line", "?")
            raise type(e)(f"{template_name}:{tpl_line}: {e}") from e

    return wrapper


def page_path_to_permalink(path: pathlib.Path) -> str:
    return "/" + str(path.relative_to(pages_path).with_suffix(""))


@dataclass
class PageData:
    src_path: pathlib.Path
    title: str
    description: str
    published_date: str
    permalink: str
    markdown_content: str
    markdown_content_as_html: str
    hidden_from_home_and_rss: bool
    is_draft: bool
    is_minipost: bool
    extra_css: str
    yyyyqq_dir: str | None
    last_major_update: str | None
    last_minor_update: str | None

    def last_update(self, include_minor: bool = True) -> str:
        if include_minor:
            return max(
                self.published_date,
                self.last_major_update or "",
                self.last_minor_update or "",
            )
        return max(self.published_date, self.last_major_update or "")


def parse_frontmatter(file: pathlib.Path) -> Tuple[dict[str, Any], str]:
    content = file.read_text(encoding="utf-8")
    if not content.startswith("+++\n"):
        raise SystemExit(f"{file} doesn't start with valid frontmatter")
    content = content.removeprefix("+++\n")
    toml_str, content = content.split("\n+++\n", 2)
    if content is None:
        raise SystemExit(f"{file} doesn't start with valid frontmatter")
    try:
        toml_dict = tomllib.loads(toml_str)
    except tomllib.TOMLDecodeError as err:
        err.args = err.args + (f"{file} doesn't start with valid frontmatter",)
        raise
    return toml_dict, content


# Responsible for:
# * Loading file and parsing frontmatter
# * Rendering markdown content as html
# * Creating a PageData instance and filling in the necessary fields in a
#   normalised form.
def parse_page(file: pathlib.Path, for_local_serve: bool) -> PageData:
    metadata, markdown_content = parse_frontmatter(file)
    T = TypeVar("T")

    known_keys = {
        "description",
        "published",
        "permalink",
        "hidden_from_home_and_rss",
        "extra_css",
    }
    for key in metadata.keys():
        if key not in known_keys:
            print(f"{file}: Warning! Frontmatter field '{key}' not recognised")

    def fm_get(key: str, ty: type[T]) -> T:
        if key not in metadata:
            raise SystemExit(f"Post {file} missing required frontmatter field '{key}'.")
        val = metadata[key]
        if not isinstance(val, ty):
            raise SystemExit(
                f"Post {file} has unexpected type in frontmatter field '{key}', expected {ty}."
            )
        return val

    def fm_determine_published_date_and_is_draft() -> Tuple[str, bool]:
        if "published" not in metadata:
            raise SystemExit(
                f"Post {file} missing required frontmatter field 'published'."
            )
        published = metadata["published"]
        if isinstance(published, datetime.date):
            return published.strftime("%Y-%m-%d"), False
        elif published == "draft":
            return datetime.datetime.now().strftime("%Y-%m-%d"), True
        else:
            raise SystemExit(
                f"{file}: Error! 'published' must be set to date or 'draft'"
            )

    def fm_get_opt(key: str, ty: type[T]) -> T | None:
        if key not in metadata:
            return None
        return fm_get(key, ty)

    description = fm_get("description", str)
    published_date, is_draft = fm_determine_published_date_and_is_draft()
    permalink = page_path_to_permalink(file)
    hidden_from_home_and_rss = fm_get_opt("hidden_from_home_and_rss", bool) or (
        is_draft and not for_local_serve
    )
    extra_css = fm_get_opt("extra_css", str) or ""
    path_part = file.parts[1]
    yyyyqq_dir = None
    if re.fullmatch(r"\d{4}q[1-4]", path_part):
        yyyyqq_dir = path_part
    is_minipost = file.stem.startswith("minipost-")
    if is_minipost and not yyyyqq_dir:
        raise SystemExit(f"{file}: Error! miniposts should be in a yyyyqq directory")

    (
        markdown_content_as_html,
        title,
        last_major_update,
        last_minor_update,
    ) = render_markdown(
        markdown_content, file, published_date, is_draft, is_minipost, yyyyqq_dir
    )

    return PageData(
        src_path=file,
        yyyyqq_dir=yyyyqq_dir,
        title=title,
        description=description,
        published_date=published_date,
        permalink=permalink,
        markdown_content=markdown_content,
        markdown_content_as_html=markdown_content_as_html,
        hidden_from_home_and_rss=hidden_from_home_and_rss,
        is_draft=is_draft,
        is_minipost=is_minipost,
        extra_css=extra_css,
        last_major_update=last_major_update,
        last_minor_update=last_minor_update,
    )


def strip_anchor(target: str) -> str:
    return target.split("#")[0]


def check_link_target(target: str, src_file: pathlib.Path) -> None:
    if target.startswith("http://") or target.startswith("https://"):
        return
    if target.startswith("//"):
        raise SystemExit(
            f"Post {src_file} has '//' protocol-relative link '{target}'. Potential typo?"
        )
    if target in ["/", "/feed.xml", "/sitemap.xml", "/robots.txt"]:
        return
    if not target.startswith("/"):
        raise SystemExit(f"Post {src_file} contains relative link '{target}'.")
    if not pathlib.Path(target[1:]).is_file():
        raise SystemExit(f"Post {src_file} links to {target}, which doesn't exist.")
    return


class MuxupRenderer(mistletoe.HTMLRenderer):  # type:ignore
    formatter = HtmlFormatter(style="xcode", noclasses=True, wrapcode=True)

    def __init__(self, *extras):  # type:ignore
        super().__init__(*extras)
        self.heading_slugs = set()

    def render_block_code(self, token: mistletoe.block_token.BlockCode) -> str:
        code = token.children[0].content  # type: ignore
        if token.language:
            try:
                lexer = get_lexer(token.language)
            except pygments.util.ClassNotFound:
                lexer = None
            if lexer:
                return highlight(code, lexer, self.formatter)  # type:ignore
        return super().render_block_code(token)  # type:ignore

    def render_heading(self, token: mistletoe.block_token.Heading) -> str:
        template = '<h{level} id="{linkid}"><a href="#{linkid}" class="anchor" aria-hidden="true" tabindex="-1"></a>{inner}</h{level}>'
        inner = self.render_inner(token)
        linkid = re.sub(r"[^a-z0-9-_]", "", inner.lower().replace(" ", "-"))
        unique_linkid = linkid
        i = 1
        while unique_linkid in self.heading_slugs:
            unique_linkid = f"{linkid}-{i}"
            i += 1
        self.heading_slugs.add(unique_linkid)
        return template.format(level=token.level, inner=inner, linkid=unique_linkid)


# Renders markdown to html, applying any necessary checks and transformations
# at the AST level. Also extracts the title (first heading). Returns a tuple
# of (rendered_markdown, title, last_major_update, last_minor_update).
def render_markdown(
    content: str,
    src_file: pathlib.Path,
    published_date: str,
    is_draft: bool,
    is_minipost: bool,
    yyyyqq_dir: str | None,
) -> tuple[str, str, str | None, str | None]:
    if is_draft:
        initial_changelog_entry = "(Draft: not yet 'published')"
    else:
        initial_changelog_entry = f"{published_date}: Initial publication date."
    changelog_opening_html = '<hr style="margin-top:1.75rem"/><details id="article-changelog"><summary><a href="#article-changelog" class="anchor" aria-hidden="true" tabindex="-1"></a>Article changelog</summary>'

    def err(desc: str) -> None:
        raise SystemExit(f"ERROR: {src_file} {desc}")

    def extract_and_remove_article_title(doc: mistletoe.Document) -> str:  # type: ignore
        if not doc.children:
            err("document is empty")
        if not isinstance(doc.children[0], mistletoe.block_token.Heading):
            err("document doesn't start with heading")
        if doc.children[0].level != 1:
            err("first heading isn't a top-level ('# Foo') heading")
        h1 = doc.children[0]
        if not h1.children:
            err("title is empty")
        if len(h1.children) > 1 or not isinstance(
            h1.children[0], mistletoe.span_token.RawText
        ):
            err("title isn't plain text")
        title = h1.children[0].content
        del doc.children[0]
        return str(title)

    def check_and_rewrite_links(doc: mistletoe.Document) -> None:  # type: ignore
        # Rewrite any /pages/foo.md links to appropriate permalinks.
        for t in mistletoe.utils.traverse(doc, klass=mistletoe.span_token.Link):  # type: ignore
            # TODO: check anchor links, including within the current document.
            stripped_target = strip_anchor(t.node.target)
            check_link_target(stripped_target, src_file)
            if stripped_target.startswith("/pages/"):
                t.node.target = page_path_to_permalink(
                    pathlib.Path(stripped_target[1:])
                )

    def process_changelog(
        doc: mistletoe.Document,  # type: ignore
        renderer: MuxupRenderer,
    ) -> tuple[str | None, str | None]:
        # Extract changelog info if present.
        changelog_heading = None
        last_major_update = None
        last_minor_update = None
        for t in mistletoe.utils.traverse(doc, klass=mistletoe.block_token.Heading):  # type: ignore
            if changelog_heading:
                err("changelog wasn't last heading")
            # TODO: error if changelog heading isn't the last
            if renderer.render_inner(t.node).lower() == "article changelog":
                if t.node.level != 2:
                    err("changelog not a '## Level 2 heading'")
                changelog_heading = t.node
        if not changelog_heading:
            return (last_major_update, last_minor_update)
        changelog_idx = doc.children.index(changelog_heading)
        if len(doc.children) < changelog_idx + 1:
            err("changelog heading has no content")
        if not isinstance(doc.children[changelog_idx + 1], mistletoe.block_token.List):
            err("changelog doesn't have list as first child")
        changelog_list = doc.children[changelog_idx + 1]
        last_li_date = None
        for li in changelog_list.children:
            li_str = renderer.render_inner(li)
            li_date = li_str[3:13]
            if li_date < published_date:
                err("changelog entry predates published date")
            if last_li_date and li_date > last_li_date:
                err("changelog entries out of order")
            last_li_date = li_date
            try:
                datetime.date.fromisoformat(li_date)
            except ValueError:
                err("changelog entry not in expected format")
            if li_str[13:].startswith(": (minor)"):
                last_minor_update = max(last_minor_update or "", li_date)
                li_text_child = li.children[0].children[0]
                li_text_child.content = (
                    li_text_child.content[:11] + li_text_child.content[19:]
                )
            else:
                last_major_update = max(last_major_update or "", li_date)
        changelog_list.children.append(
            mistletoe.Document(f"* {initial_changelog_entry}").children[0].children[0]  # type: ignore
        )
        doc.children[changelog_idx] = mistletoe.block_token.HTMLBlock(  # type: ignore
            changelog_opening_html
        )
        doc.children.insert(
            changelog_idx + 2,
            mistletoe.block_token.HTMLBlock("</details>"),  # type: ignore
        )
        # Add list item for creation of the article.
        return (last_major_update, last_minor_update)

    with MuxupRenderer() as renderer:  # type:ignore
        doc = mistletoe.Document(content)  # type: ignore
        title = extract_and_remove_article_title(doc)
        if is_minipost:
            title = f"Minipost: {title}"
        check_and_rewrite_links(doc)
        last_major_update, last_minor_update = process_changelog(doc, renderer)
        markdown_content_as_html = renderer.render(doc)
        if not (last_major_update or last_minor_update):
            markdown_content_as_html = f"""\
{markdown_content_as_html}
{changelog_opening_html}
<ul>
<li>{initial_changelog_entry}</li>
</ul>
</details>"""

        return (markdown_content_as_html, title, last_major_update, last_minor_update)


def quarter_for_date(d: datetime.date) -> int:
    return ((d.month - 1) // 3) + 1


def generate_article_meta(pd: PageData) -> str:
    ret = []
    if pd.yyyyqq_dir:
        ret.append(f'<span title="{pd.published_date}">{pd.yyyyqq_dir.upper()}</span>.')
    if pd.last_major_update or pd.last_minor_update:
        update = datetime.date.fromisoformat(pd.last_update())
        today = datetime.date.today()
        days_since = (today - update).days
        if days_since < 30:
            update_formatted = update.strftime("%d %b %Y")
        elif days_since < 90:
            update_formatted = update.strftime("%b %Y")
        else:
            update_formatted = f"{update.year}Q{quarter_for_date(update)}"
        ret.append(
            f'Last update <span title="{pd.last_update()}">{update_formatted}</span>.'
        )
    if not pd.yyyyqq_dir or pd.last_major_update or pd.last_minor_update:
        ret.append(
            """\
<a href="#article-changelog"\
 onclick="document.querySelector('#article-changelog').setAttribute('open', true)">\
History↓</a>"""
        )
    return (" ").join(ret)


written_dest_files = set()


def atomic_write_text(path: pathlib.Path, data: str, encoding: str = "utf-8") -> None:
    written_dest_files.add(path)
    tmp_path = path.with_name(path.name + "~")
    tmp_path.write_text(data, encoding=encoding)
    os.rename(tmp_path, path)


# Core logic


# Read all pages markdown files, parse frontmatter, and generate HTML.

css_template = compile_template(
    pathlib.Path("templates/style.css.tpl").read_text(), "templates/style.css.tpl"
)


def generate_css_string(
    page_type: str, extra_css: str = "", html_for_gated_css_check: str = ""
) -> str:
    return css_template(
        page_type=page_type, extra_css=extra_css, target_html=html_for_gated_css_check
    )


page_template = compile_template(
    pathlib.Path("templates/page.html.tpl").read_text(), "templates/page.html.tpl"
)


def build_single_page(
    file: pathlib.Path,
    out_dir: pathlib.Path,
    for_local_serve: bool,
    minified_article_js: str,
) -> Tuple[PageData, pathlib.Path]:
    pd = parse_page(file, for_local_serve)
    suffix = "" if for_local_serve else ".html"
    out_path = out_dir / file.relative_to(pages_path).with_suffix(suffix)
    if not out_path.parent.exists():
        out_path.parent.mkdir(parents=True)
    page_content = page_template(
        h=html.escape,
        base_url=base_url,
        favicon_svg=favicon_svg,
        pd=pd,
        minified_article_js=minified_article_js,
        opengraph_image=f"https://v1.screenshot.11ty.dev/{urllib.parse.quote_plus(base_url + pd.permalink)}/opengraph/ar/bigger",
        css=generate_css_string("article", pd.extra_css, pd.markdown_content_as_html),
        article_meta=generate_article_meta(pd),
    )
    atomic_write_text(out_path, page_content)
    return pd, out_path


def build_pages(
    out_dir: pathlib.Path, for_local_serve: bool, minified_article_js: str
) -> list[PageData]:
    pages_data: list[PageData] = []

    with concurrent.futures.ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
        futures = [
            executor.submit(
                build_single_page, file, out_dir, for_local_serve, minified_article_js
            )
            for file in sorted(pages_path.rglob("*.md"))
        ]
        for future in concurrent.futures.as_completed(futures):
            pd, out_path = future.result()
            pages_data.append(pd)
            # Need to manually add to written_dest_files as changes from
            # within the process won't be reflected here.
            written_dest_files.add(out_path)
    return pages_data


def build_front_page(
    pages_data: list[PageData], out_dir: pathlib.Path, minified_home_js: str
) -> None:
    out_path = out_dir / "index.html"
    num_card_grid_entries = sum(
        1 for pd in pages_data if not pd.hidden_from_home_and_rss
    )
    pages_data.sort(key=lambda pd: (pd.last_update(False), pd.permalink), reverse=True)
    sorted_filtered_pages_for_cards = filter(
        lambda pd: not (pd.hidden_from_home_and_rss or pd.is_minipost), pages_data
    )
    sorted_filtered_minipost_pages = filter(
        lambda pd: not pd.hidden_from_home_and_rss and pd.is_minipost, pages_data
    )

    home_template = compile_template(
        pathlib.Path("templates/home.html.tpl").read_text(), "templates/home.html.tpl"
    )
    home_content = home_template(
        h=html.escape,
        base_url=base_url,
        opengraph_image=f"https://v1.screenshot.11ty.dev/{urllib.parse.quote_plus(base_url)}/opengraph/ar/bigger/_{num_card_grid_entries}",
        favicon_svg=favicon_svg,
        css=generate_css_string("home"),
        sorted_filtered_pages_for_cards=sorted_filtered_pages_for_cards,
        sorted_filtered_minipost_pages=sorted_filtered_minipost_pages,
        minified_home_js=minified_home_js,
    )

    atomic_write_text(out_path, home_content)


def build_robots_txt(out_dir: pathlib.Path) -> None:
    out_path = out_dir / "robots.txt"
    template = compile_template(
        pathlib.Path("templates/robots.txt.tpl").read_text(), "templates/robots.txt.tpl"
    )
    content = template(base_url=base_url)
    atomic_write_text(out_path, content)


def build_sitemap_xml(pages_data: list[PageData], out_dir: pathlib.Path) -> None:
    # Simple iteration over frontmatter in order of last_updated again (but
    # unlike front page, by last minor update)
    out_path = out_dir / "sitemap.xml"
    template = compile_template(
        pathlib.Path("templates/sitemap.xml.tpl").read_text(),
        "templates/sitemap.xml.tpl",
    )
    filtered_pages_data = list(filter(lambda pd: not pd.is_draft, pages_data))
    filtered_pages_data.sort(
        key=lambda pd: (pd.last_update(), pd.permalink), reverse=True
    )
    content = template(
        base_url=base_url,
        front_page_lastmod=max(pd.last_update(False) for pd in filtered_pages_data),
        filtered_pages_data=filtered_pages_data,
    )
    atomic_write_text(out_path, content)


def build_feed_xml(pages_data: list[PageData], out_dir: pathlib.Path) -> None:
    out_path = out_dir / "feed.xml"
    template = compile_template(
        pathlib.Path("templates/feed.xml.tpl").read_text(), "templates/feed.xml.tpl"
    )
    pages_data.sort(key=lambda pd: (pd.last_update(False), pd.permalink), reverse=True)
    filtered_pages_data = list(
        filter(lambda pd: not pd.hidden_from_home_and_rss, pages_data)
    )

    def replace_rel_with_abs_links(string: str) -> str:
        return string.replace("href=/", f"href={base_url}/").replace(
            "src=/", "src={base_url}/"
        )

    content = template(
        base_url=base_url,
        h=html.escape,
        filtered_pages_data=filtered_pages_data,
        replace_rel_with_abs_links=replace_rel_with_abs_links,
    )
    atomic_write_text(out_path, content)


def build_static_subdir(out_dir: pathlib.Path) -> None:
    tmp_dest_static_path = out_dir / "static~"
    dest_static_path = out_dir / "static"
    dest_static_path.mkdir(exist_ok=True)
    src_static_path = pathlib.Path("static")
    if tmp_dest_static_path.exists():
        shutil.rmtree(tmp_dest_static_path)

    def copy_and_track(src: str, dst: str) -> None:
        written_dest_files.add(out_dir / src)
        shutil.copy2(src, dst)

    shutil.copytree(src_static_path, tmp_dest_static_path, copy_function=copy_and_track)

    subprocess.run(["exch", tmp_dest_static_path, dest_static_path], check=True)
    shutil.rmtree(tmp_dest_static_path)


def generate_minified_js(
    common_js_frag: str, home_js_frag: str, article_js_frag: str
) -> Tuple[str, str]:
    def minify(in_js: str) -> str:
        return subprocess.run(
            ["terser", "--mangle-props", "--toplevel"],
            input=in_js,
            capture_output=True,
            check=True,
            encoding="utf-8",
        ).stdout

    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
        home_future = executor.submit(minify, common_js_frag + home_js_frag)
        article_future = executor.submit(minify, common_js_frag + article_js_frag)
        return (home_future.result(), article_future.result())


def rebuild(for_local_serve: bool = False, with_simple_reload: bool = False) -> None:
    print("Starting rebuild")
    out_dir = staged_path
    if for_local_serve:
        print("Building for local serve usage")
        out_dir = local_serve_path
    if not out_dir.exists():
        out_dir.mkdir()

    preexisting_dest_files = set()
    for dirpath, dirnames, filenames in os.walk(out_dir):
        for fn in filenames:
            preexisting_dest_files.add(pathlib.Path(dirpath) / fn)
        for dn in dirnames:
            child_path = pathlib.Path(dirpath) / dn
            if child_path.is_symlink():
                preexisting_dest_files.add(pathlib.Path(dirpath) / dn)

    footer_data_path = pathlib.Path("footer_drawings.json")
    footer_data = json.loads(footer_data_path.read_text(encoding="utf-8"))
    num_footer_images = len(footer_data)

    common_js_frag = (
        pathlib.Path("fragments/common.js")
        .read_text(encoding="utf-8")
        .replace("NUM_FOOTER_IMAGES", str(num_footer_images))
    )
    refresh_js_frag = pathlib.Path("fragments/simple-reload.js").read_text(
        encoding="utf-8"
    )
    check_error_js_frag = pathlib.Path("fragments/check-error.js").read_text(
        encoding="utf-8"
    )
    if with_simple_reload:
        common_js_frag = f"{common_js_frag}\n{refresh_js_frag}"
    if for_local_serve:
        common_js_frag = f"{common_js_frag}\n{check_error_js_frag}"
    home_js_frag = pathlib.Path("fragments/home.js").read_text(encoding="utf-8")
    article_js_frag = pathlib.Path("fragments/article.js").read_text(encoding="utf-8")
    minified_home_js, minified_article_js = generate_minified_js(
        common_js_frag, home_js_frag, article_js_frag
    )

    pages_data = build_pages(out_dir, for_local_serve, minified_article_js)
    build_front_page(pages_data, out_dir, minified_home_js)
    build_robots_txt(out_dir)
    build_sitemap_xml(pages_data, out_dir)
    build_feed_xml(pages_data, out_dir)
    build_static_subdir(out_dir)

    files_to_delete = preexisting_dest_files - written_dest_files
    for path in files_to_delete:
        print(f"Removing {path} as it wasn't regenerated")
        path.unlink()
    for dirpath, dirnames, filenames in os.walk(out_dir, topdown=False):
        for dn in dirnames:
            path = pathlib.Path(dirpath) / dn
            if any(os.scandir(path)):
                continue
            print(f"Removing empty path {path}")
            os.rmdir(pathlib.Path(dirpath) / dn)

    print("Rebuild finished")


def deploy() -> None:
    if not staged_path.is_dir():
        raise SystemExit(f"{staged_path} folder not present")

    if deployed_path.exists():
        shutil.rmtree(deployed_path)
    shutil.copytree(staged_path, deployed_path)

    print("Compressing files with brotli")
    subprocess.run(
        f"find {deployed_path} \\( -name '*.html' -o -name '*.svg' \\) -print0 | "
        "xargs -0 -P$(nproc) brotli -q 11 -k -f",
        shell=True,
        check=True,
    )

    try:
        deploy_target = subprocess.run(
            ["git", "config", "get", "muxup.deploytarget"],
            encoding="utf-8",
            check=True,
            capture_output=True,
        ).stdout.strip()
    except subprocess.CalledProcessError:
        raise SystemExit(
            "Failed to get deploy target from git config.\n"
            "Set it with: git config set muxup.deploytarget 'user@host:/path/to/target'"
        )

    print("Rsyncing")
    subprocess.run(
        ["rsync", "-avcz", "--no-t", "--delete", f"{deployed_path}/", deploy_target],
        check=True,
    )


def diff() -> None:
    print(f"Diff between '{staged_path}/' and '{deployed_path}/':")
    sys.exit(
        subprocess.run(
            ["diff", "--color", "--exclude=*.br", "-r", deployed_path, staged_path]
        ).returncode
    )


def serve(with_simple_reload: bool = False) -> None:
    def rebuild_loop() -> None:
        while True:
            paths = (
                list(pathlib.Path("fragments").rglob("*"))
                + list(pages_path.rglob("*"))
                + list(pathlib.Path("static").rglob("*"))
                + list(pathlib.Path("templates").rglob("*"))
                + [pathlib.Path("gen")]
            )
            filenames = [str(p) for p in paths if p.is_file()]
            subprocess.run(
                ["entr", "-d", "./gen", "build", "--for-local-serve"]
                + (["--with-simple-reload"] if with_simple_reload else []),
                input="\n".join(filenames),
                encoding="utf-8",
            )

    print("Launching watcher")
    threading.Thread(target=rebuild_loop, daemon=True).start()
    print("Starting server. See http://localhost:5500")
    try:
        # TODO: Relying on default-mimetype is a bit of a hack. Ideally
        # darkhttpd would support pretty URLs directly like requested in
        # <https://github.com/emikulic/darkhttpd/issues/71>.
        server = subprocess.Popen(
            [
                "darkhttpd",
                "./local_serve",
                "--port",
                "5500",
                "--addr",
                "127.0.0.1",
                "--default-mimetype",
                "text/html",
            ]
        )
        server.wait()
    except KeyboardInterrupt:
        print("Ctrl-C received, shutting down server")
        server.terminate()
        server.wait()
        raise SystemExit


def get_draft_articles() -> Tuple[List[pathlib.Path], List[pathlib.Path]]:
    draft_articles = []
    non_draft_articles = []
    for file in pages_path.rglob("*.md"):
        metadata, _ = parse_frontmatter(file)
        if metadata["published"] == "draft":
            draft_articles.append(file)
        else:
            non_draft_articles.append(file)
    return draft_articles, non_draft_articles


def status() -> None:
    res = subprocess.run(
        ["diff", "--color", "--exclude=*.br", "-q", "-r", deployed_path, staged_path],
        capture_output=True,
        encoding="utf-8",
    )
    if res.returncode > 1:
        raise SystemExit("Unexpected failure executing diff")
    num_differing_files = res.stdout.count("\n")
    if num_differing_files == 0:
        print("No differences between 'deployed' and 'staged'\n")
    else:
        print(f"{num_differing_files} differing files between 'deployed' and 'staged':")
        print(res.stdout)

    draft_articles, non_draft_articles = get_draft_articles()

    num_draft_articles = len(draft_articles)
    if num_draft_articles == 0:
        print("0 draft articles found\n")
    else:
        print(f"{num_draft_articles} draft articles found:")
        for file in sorted(draft_articles):
            print(file)
        print()

    print(f"{len(non_draft_articles)} published articles found.")


def commit_untracked() -> None:
    def exec(*args: Any, **kwargs: Any) -> tuple[str, int]:
        kwargs.setdefault("encoding", "utf-8")
        kwargs.setdefault("capture_output", True)
        kwargs.setdefault("check", True)

        result = subprocess.run(*args, **kwargs)
        return result.stdout.rstrip("\n"), result.returncode

    result, _ = exec(["git", "status", "-uall", "--porcelain", "-z"])
    untracked_files = []
    entries = result.split("\0")
    for entry in entries:
        if entry.startswith("??"):
            untracked_files.append(entry[3:])

    if len(untracked_files) == 0:
        print("No untracked files to commit.")
        return

    bak_branch = "refs/heads/bak"
    show_ref_result, returncode = exec(
        ["git", "show-ref", "--verify", bak_branch], check=False
    )
    if returncode != 0:
        print("Branch {back_branch} doesn't yet exist - it will be created")
        parent_commit = ""
        parent_commit_tree = None
        commit_message = "Initial commit of untracked files"
        extra_write_tree_args = []
    else:
        parent_commit = show_ref_result.split()[0]
        parent_commit_tree, _ = exec(["git", "rev-parse", f"{parent_commit}^{{tree}}"])
        commit_message = "Update untracked files"
        extra_write_tree_args = ["-p", parent_commit]

    # Use a temporary index in order to create a commit. Add any untracked
    # files to the index, create a tree object based on the index state, and
    # finally create a commit using that tree object.
    temp_index = pathlib.Path(".drafts.gitindex.tmp")
    atexit.register(lambda: temp_index.unlink(missing_ok=True))
    git_env = os.environ.copy()
    git_env["GIT_INDEX_FILE"] = str(temp_index)
    nul_terminated_untracked_files = "\0".join(file for file in untracked_files)
    exec(
        ["git", "update-index", "--add", "-z", "--stdin"],
        input=nul_terminated_untracked_files,
        env=git_env,
    )
    tree_sha, _ = exec(["git", "write-tree"], env=git_env)
    if tree_sha == parent_commit_tree:
        print("Untracked files are unchanged vs last commit - nothing to do.")
        return
    commit_sha, _ = exec(
        ["git", "commit-tree", tree_sha] + extra_write_tree_args,
        input=commit_message,
    )
    exec(["git", "update-ref", bak_branch, commit_sha])

    diff_stat, _ = exec(["git", "show", "--stat", "--format=", commit_sha])

    print(f"Backup branch '{bak_branch}' updated successfully.")
    print(f"Created commit {commit_sha} with the following modifications:")
    print(diff_stat)


# Command line handling

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="muxup.com static site generator", add_help=False
    )
    subparsers = parser.add_subparsers(
        dest="command", help="Available commands", required=True
    )
    build_parser = subparsers.add_parser("build", help="Rebuild the site and exit")
    build_parser.add_argument(
        "--for-local-serve",
        action="store_true",
        help="Generate files in format appropriate for local http server",
    )
    build_parser.add_argument(
        "--with-simple-reload",
        action="store_true",
        help="Inject simple-reload.js",
    )
    commit_untracked_parser = subparsers.add_parser(
        "commit-untracked",
        help="Commit any untracked files (e.g. drafts) directly to the 'bak' branch",
    )
    deploy_parser = subparsers.add_parser(
        "deploy", help="Deploy the already-built files"
    )
    diff_parser = subparsers.add_parser(
        "diff", help="Diff between current staged and deployed directories"
    )
    help_parser = subparsers.add_parser("help", help="Show this help message")
    serve_parser = subparsers.add_parser(
        "serve", help="Serve locally, rebuilding upon edit"
    )
    serve_parser.add_argument(
        "--with-simple-reload",
        action="store_true",
        help="Inject simple-reload.js",
    )
    status_parser = subparsers.add_parser(
        "status",
        help="Print status of site (e.g. number of draft posts, if there's a staged vs deployed diff)",
    )
    args = parser.parse_args()

    if args.command == "build":
        rebuild(
            for_local_serve=args.for_local_serve,
            with_simple_reload=args.with_simple_reload,
        )
    elif args.command == "commit-untracked":
        commit_untracked()
    elif args.command == "deploy":
        deploy()
    elif args.command == "diff":
        diff()
    elif args.command == "help":
        parser.print_help()
    elif args.command == "serve":
        serve(with_simple_reload=args.with_simple_reload)
    elif args.command == "status":
        status()